def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' need to strip that off for key, value in env.iteritems(): if ("wset_" in key and (value == 'True' or env['all_waccm_sets'] == 'True')): requested_plot_sets.append(key) elif ("cset_" in key and (value == 'True' or env['all_chem_sets'] == 'True')): requested_plot_sets.append(key) elif ("set_" in key and (value == 'True' or env['all_sets'] == 'True')): if ("wset_" not in key and "cset_" not in key): requested_plot_sets.append(key) scomm.sync() # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update( atm_diags_plot_factory.atmosphereDiagnosticPlotFactory( plot_set, env)) for plot_id, plot_class in requested_plots.iteritems(): if hasattr(plot_class, 'weight'): factor = plot_class.weight else: factor = 1 plots_weights.append( (plot_id, len(plot_class.expectedPlots) * factor)) # partition based on the number of plots each set will create local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script #for plot_id,plot_class in local_plot_list.interitems(): timer.start(str(scomm.get_rank()) + "ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank()) + plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) plot_class.plot_env['NCDF_MODE'] = 'write' plot_class.plot_env['VAR_MODE'] = 'write' timer.stop(str(scomm.get_rank()) + plot_set) timer.stop(str(scomm.get_rank()) + "ncl total time on task") scomm.sync() print(timer.get_all_times()) w = 0 for p in plots_weights: if p[0] in local_plot_list: w = w + p[1] print(str(scomm.get_rank()) + ' weight:' + str(w)) # set html files if scomm.is_manager(): env['HTML_HOME'] = env['DIAG_HOME'] + '/html/model1-model2/' # Get web dir name and create it if it does not exist #web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename']) web_dir = env['test_path_diag'] #if not os.path.exists(web_dir): # os.makedirs(web_dir) # Copy over some files needed by web pages if not os.path.exists(web_dir + '/images'): os.mkdir(web_dir + '/images') diag_imgs = glob.glob(env['DIAG_HOME'] + '/html/images/*') for img in diag_imgs: shutil.copy(img, web_dir + '/images/') # Create set dirs, copy plots to set dir, and create html file for set requested_plot_sets.append( 'sets') # Add 'sets' to create top level html files for plot_set in requested_plot_sets: if 'set_5' == plot_set or 'set_6' == plot_set: glob_set = plot_set.replace('_', '') plot_set = 'set5_6' elif 'set_1' == plot_set: glob_set = 'table_' plot_set = plot_set.replace('_', '') elif 'sets' == plot_set: set_dir = web_dir + '/' else: plot_set = plot_set.replace('_', '') glob_set = plot_set if 'sets' not in plot_set: #'sets' is top level, don't create directory or copy images files set_dir = web_dir + '/' + plot_set # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir glob_string = env['test_path_diag'] + '/' + glob_set + '*.*' imgs = glob.glob(glob_string) if imgs > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) # Copy/Process html files if 'sets' in plot_set: orig_html = env['HTML_HOME'] + '/' + plot_set else: orig_html = env[ 'HTML_HOME'] + '/' + plot_set + '/' + plot_set create_atm_html.create_plotset_html(orig_html, set_dir, plot_set, env, 'model_vs_model') # Remove any plotvar netcdf files that exists in the diag directory if env['save_ncdfs'] == 'False': cesmEnvLib.purge(env['test_path_diag'], '.*\.nc') cesmEnvLib.purge(env['test_path_diag'], '/station_ids') # move all the plots to the diag_path with the years appended to the path endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1 endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1 diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format( env['OUTPUT_ROOT_PATH'], env['test_casename'], env['test_first_yr'], str(endYr1), env['cntl_casename'], env['cntl_first_yr'], str(endYr2)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format( self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t', 'true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk('diag_path'): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f', 'false']: print( 'WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}' .format(diag_path, web_dir)) diag_path = web_dir move_files = False # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_model renaming web files') os.rename(web_dir, diag_path) except OSError as e: print('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir # setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH']) key = 'ATMDIAG_WEBDIR_{0}'.format(self._name) value = diag_path ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format( env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'atm', key, value) except: print( 'WARNING atm model_vs_model unable to write {0}={1} to {2}' .format(key, value, web_file)) print( '*******************************************************************************' ) print( 'Successfully completed generating atmosphere diagnostics model vs. model plots' ) print( '*******************************************************************************' )
def check_prerequisites(self, env): """ check prerequisites """ print(' Checking prerequisites for : {0}'.format( self.__class__.__name__)) self._name = '{0}_{1}'.format(self._name, env['CNTRLCASE']) super(modelVsControl, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T', 'TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # setup prerequisites for the model # setup the gridfile based on the resolution and levels os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format( env['DIAGOBSROOT'], env['RESOLUTION']) if env['VERTICAL'] == '42': os.environ[ 'gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format( env['DIAGOBSROOT'], env['RESOLUTION']) if env['VERTICAL'] == '62': os.environ[ 'gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format( env['DIAGOBSROOT'], env['RESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read') if not rc: raise OSError(err_msg) env['GRIDFILE'] = os.environ['gridfile'] # check the resolution and decide if some plot modules should be turned off if env['RESOLUTION'] == 'tx0.1v2': env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE' env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE' # create the global zonal average file used by most of the plotting classes print(' model vs. control - calling create_za') diagUtilsLib.create_za(env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env) # setup prerequisites for the model control control = True env['CNTRL_MAVGFILE'], env[ 'CNTRL_TAVGFILE'] = diagUtilsLib.createLinks( env['CNTRLYEAR0'], env['CNTRLYEAR1'], env['CNTRLTAVGDIR'], env['WORKDIR'], env['CNTRLCASE'], control) env['CNTRLFILE'] = env['CNTRL_TAVGFILE'] # setup the gridfile based on the resolution and vertical levels os.environ[ 'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format( env['DIAGOBSROOT'], env['CNTRLRESOLUTION']) if env['VERTICAL'] == '42': os.environ[ 'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format( env['DIAGOBSROOT'], env['CNTRLRESOLUTION']) if env['VERTICAL'] == '62': os.environ[ 'gridfilecntrl'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format( env['DIAGOBSROOT'], env['CNTRLRESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfilecntrl'], 'read') if not rc: raise OSError(err_msg) env['GRIDFILECNTRL'] = os.environ['gridfilecntrl'] # check the resolution and decide if some plot modules should be turned off if env['CNTRLRESOLUTION'] == 'tx0.1v2': env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE' env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE' # create the control global zonal average file used by most of the plotting classes print(' model vs. control - calling create_za for control run') diagUtilsLib.create_za(env['WORKDIR'], env['CNTRL_TAVGFILE'], env['GRIDFILECNTRL'], env['TOOLPATH'], env) return env
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format(self.__class__.__name__)) super(modelTimeseries, self).check_prerequisites(env) # chdir into the working directory os.chdir(env['WORKDIR']) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') cesmEnvLib.purge(env['WORKDIR'], '.*\.log\.*') cesmEnvLib.purge(env['WORKDIR'], '.*\.pop\.d.\.*') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # set the OBSROOT env['OBSROOT'] = env['OBSROOTPATH'] # check the resolution and decide if some plot modules should be turned off if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') : env['MTS_PM_MOCANN'] = os.environ['PM_MOCANN'] = 'FALSE' env['MTS_PM_MOCMON'] = os.environ['PM_MOCMON'] = 'FALSE' # check if cpl log file path is defined if len(env['CPLLOGFILEPATH']) == 0: # print a message that the cpl log path isn't defined and turn off CPLLOG plot module print('model timeseries - CPLLOGFILEPATH is undefined. Disabling MTS_PM_CPLLOG module') env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE' else: # check that cpl log files exist and gunzip them if necessary initcplLogs = cplLogs = list() initCplLogs = glob.glob('{0}/cpl.log.*'.format(env['CPLLOGFILEPATH'])) if len(initCplLogs) > 0: for cplLog in initCplLogs: logFileList = cplLog.split('/') cplLogFile = logFileList[-1] shutil.copy2(cplLog, '{0}/{1}'.format(env['WORKDIR'],cplLogFile)) # gunzip the cplLog in the workdir if cplLogFile.lower().find('.gz') != -1: cplLog_gunzip = cplLogFile[:-3] inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],cplLogFile), 'rb') outFile = open('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip), 'wb') outFile.write( inFile.read() ) inFile.close() outFile.close() # append the gunzipped cpl log file to the cplLogs list cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip)) # remove the original .gz file in the workdir os.remove('{0}/{1}'.format(env['WORKDIR'],cplLogFile)) else: # append the original gunzipped cpl log file to the cplLogs list cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLogFile)) # parse the cpllog depending on the coupler version - default to 7b print('model_timeseries: setting up heat and freshwater awk calls with cplLogs = {0}'.format(cplLogs)) heatFile = 'cplheatbudget' freshWaterFile = 'cplfwbudget' cplVersion = 'cpl7b' env['ntailht'] = os.environ['ntailht'] = '22' env['ntailfw'] = os.environ['ntailfw'] = '16' if '7' == env['TS_CPL'] or '6' == env['TS_CPL']: cplVersion = 'cpl{0}'.format(env['TS_CPL']) env['ntailht'] = os.environ['ntailht'] = '21' env['ntailfw'] = os.environ['ntailfw'] = '16' # expand the cpl.log* into a list cplLogs.sort() cplLogsString = ' '.join(cplLogs) # define the awk scripts to parse the cpllog file heatPath = '{0}/process_{1}_logfiles_heat.awk'.format(env['TOOLPATH'], cplVersion) heatPath = os.path.abspath(heatPath) fwPath = '{0}/process_{1}_logfiles_fw.awk'.format(env['TOOLPATH'], cplVersion) fwPath = os.path.abspath(fwPath) heatCmd = '{0} y0={1} y1={2} {3}'.format(heatPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ') freshWaterCmd = '{0} y0={1} y1={2} {3}'.format(fwPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ') # run the awk scripts to generate the .txt files from the cpllogs cmdList = [ (heatCmd, heatFile, env['ntailht']), (freshWaterCmd, freshWaterFile, env['ntailfw']) ] for cmd in cmdList: outFile = '{0}.txt'.format(cmd[1]) with open (outFile, 'w') as results: try: subprocess.check_call(cmd[0], stdout=results, env=env) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) rc, err_msg = cesmEnvLib.checkFile(outFile, 'read') if rc: # get the tail of the .txt file and redirect to a .asc file for the web ascFile = '{0}.asc'.format(cmd[1]) with open (ascFile, 'w') as results: try: # TODO - read the .txt in and write just the lines needed to avoid subprocess call tailCmd = 'tail -{0} {1}.txt'.format(cmd[2], cmd[1]).split(' ') subprocess.check_call(tailCmd, stdout=results, env=env) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model timeseries - Coupler logs do not exist. Disabling MTS_PM_CPLLOG module') env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE' # check if ocn log files exist if len(env['OCNLOGFILEPATH']) == 0: # print a message that the ocn log path isn't defined and turn off POPLOG plot module print('model timeseries - OCNLOGFILEPATH is undefined. Disabling MTS_PM_YPOPLOG module') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' else: # check that ocn log files exist and gunzip them if necessary initOcnLogs = ocnLogs = list() initOcnLogs = glob.glob('{0}/ocn.log.*'.format(env['OCNLOGFILEPATH'])) if len(initOcnLogs) > 0: for ocnLog in initOcnLogs: logFileList = ocnLog.split('/') ocnLogFile = logFileList[-1] shutil.copy2(ocnLog, '{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) # gunzip the ocnLog in the workdir if ocnLogFile.lower().find('.gz') != -1: ocnLog_gunzip = ocnLogFile[:-3] inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],ocnLogFile), 'rb') outFile = open('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip), 'wb') outFile.write( inFile.read() ) inFile.close() outFile.close() # append the gunzipped ocn log file to the ocnLogs list ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip)) # remove the original .gz file in the workdir os.remove('{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) else: # append the original gunzipped ocn log file to the ocnLogs list ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) # expand the ocn.log* into a list ocnLogs.sort() ocnLogsString = ' '.join(ocnLogs) # define the awk script to parse the ocn log files globalDiagAwkPath = '{0}/process_pop2_logfiles.globaldiag.awk'.format(env['TOOLPATH']) globalDiagAwkCmd = '{0} {1}'.format(globalDiagAwkPath, ocnLogsString).split(' ') print('model_timeseries: globalDiagAwkCmd = {0}'.format(globalDiagAwkCmd)) # run the awk scripts to generate the .txt files from the ocn logs try: subprocess.check_call(globalDiagAwkCmd) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model timeseries - Ocean logs do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' # check if dt files exist if len(env['DTFILEPATH']) == 0: # print a message that the dt file path isn't defined and turn off POPLOG plot module print('model timeseries - DTFILEPATH is undefined. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' else: # check that dt files exist dtFiles = list() dtFiles = glob.glob('{0}/{1}.pop.dt.*'.format(env['DTFILEPATH'], env['CASE'])) print('dtFiles = {0}'.format(dtFiles)) if len(dtFiles) > 0: for dtFile in dtFiles: logFileList = dtFile.split('/') dtLogFile = logFileList[-1] shutil.copy2(dtFile, '{0}/{1}'.format(env['WORKDIR'],dtLogFile)) # expand the *.dt.* into a list dtFiles.sort() dtFilesString = ' '.join(dtFiles) # define the awk script to parse the dt log files dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH']) dtFilesAwkCmd = '{0} {1}'.format(dtFilesAwkPath, dtFilesString).split(' ') print('model_timeseries: dtFilesAwkCmd = {0}'.format(dtFilesAwkCmd)) # run the awk scripts to generate the .txt files from the dt log files try: subprocess.check_call(dtFilesAwkCmd) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model_timeseries - ocean dt files do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' return env
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' need to strip that off for key, value in env.iteritems(): if ("wset_"in key and (value == 'True' or env['all_waccm_sets'] == 'True')): requested_plot_sets.append(key) elif ("cset_"in key and (value == 'True' or env['all_chem_sets'] == 'True')): requested_plot_sets.append(key) elif ("set_" in key and (value == 'True' or env['all_sets'] == 'True')): if ("wset_" not in key and "cset_" not in key): requested_plot_sets.append(key) scomm.sync() # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update(atm_diags_plot_factory.atmosphereDiagnosticPlotFactory(plot_set,env)) for plot_id,plot_class in requested_plots.iteritems(): if hasattr(plot_class, 'weight'): factor = plot_class.weight else: factor = 1 plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor)) # partition based on the number of plots each set will create local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script #for plot_id,plot_class in local_plot_list.interitems(): timer.start(str(scomm.get_rank())+"ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank())+plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call plot_class.check_prerequisites(env) # Stringify the env dictionary for name,value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: diagUtilsLib.generate_ncl_plots(plot_class.plot_env,script) plot_class.plot_env['NCDF_MODE'] = 'write' plot_class.plot_env['VAR_MODE'] = 'write' timer.stop(str(scomm.get_rank())+plot_set) timer.stop(str(scomm.get_rank())+"ncl total time on task") scomm.sync() print(timer.get_all_times()) w = 0 for p in plots_weights: if p[0] in local_plot_list: w = w + p[1] print(str(scomm.get_rank())+' weight:'+str(w)) # set html files if scomm.is_manager(): env['HTML_HOME'] = env['DIAG_HOME']+'/html/model1-model2/' # Get web dir name and create it if it does not exist #web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename']) web_dir = env['test_path_diag'] #if not os.path.exists(web_dir): # os.makedirs(web_dir) # Copy over some files needed by web pages if not os.path.exists(web_dir+'/images'): os.mkdir(web_dir+'/images') diag_imgs = glob.glob(env['DIAG_HOME']+'/html/images/*') for img in diag_imgs: shutil.copy(img,web_dir+'/images/') # Create set dirs, copy plots to set dir, and create html file for set requested_plot_sets.append('sets') # Add 'sets' to create top level html files glob_set = list() for plot_set in requested_plot_sets: if 'set_5' == plot_set or 'set_6' == plot_set: glob_set.append(plot_set.replace('_','')) plot_set = 'set5_6' elif 'cset_1' == plot_set: glob_set.append('table_soa') glob_set.append('table_chem') plot_set = plot_set.replace('_','') elif 'set_1' == plot_set: glob_set.append('table_GLBL') glob_set.append('table_NEXT') glob_set.append('table_SEXT') glob_set.append('table_TROP') plot_set = plot_set.replace('_','') elif 'sets' == plot_set: set_dir = web_dir + '/' else: glob_set.append(plot_set.replace('_','')) plot_set = plot_set.replace('_','') if 'sets' not in plot_set: #'sets' is top level, don't create directory or copy images files set_dir = web_dir + '/' + plot_set # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir for gs in glob_set: glob_string = env['test_path_diag']+'/'+gs+'*.*' imgs = glob.glob(glob_string) if imgs > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img,new_fn) # Copy/Process html files if 'sets' in plot_set: orig_html = env['HTML_HOME']+'/'+plot_set else: orig_html = env['HTML_HOME']+'/'+plot_set+'/'+plot_set create_atm_html.create_plotset_html(orig_html,set_dir,plot_set,env,'model_vs_model') # Remove any plotvar netcdf files that exists in the diag directory if env['save_ncdfs'] == 'False': cesmEnvLib.purge(env['test_path_diag'], '.*\.nc') cesmEnvLib.purge(env['test_path_diag'], '/station_ids') # move all the plots to the diag_path with the years appended to the path endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1 endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1 diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(env['OUTPUT_ROOT_PATH'], env['test_casename'], env['test_first_yr'], str(endYr1), env['cntl_casename'], env['cntl_first_yr'], str(endYr2)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format(self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t','true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk(diag_path): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f','false']: print('WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'.format(diag_path, web_dir)) diag_path = web_dir move_files = False print('DEBUG: model vs. model web_dir = {0}'.format(web_dir)) print('DEBUG: model vs. model diag_path = {0}'.format(diag_path)) # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_model renaming web files') os.rename(web_dir, diag_path) except OSError as e: print ('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir # setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH']) key = 'ATMDIAG_WEBDIR_{0}'.format(self._name) value = diag_path web_file = '{0}/web_dirs/{1}.{2}'.format(env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'atm', key, value) except: print('WARNING atm model_vs_model unable to write {0}={1} to {2}'.format(key, value, web_file)) print('*******************************************************************************') print('Successfully completed generating atmosphere diagnostics model vs. model plots') print('*******************************************************************************')
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format(self.__class__.__name__)) super(modelVsObs, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # setup the gridfile based on the resolution ## os.environ['gridfile'] = '{0}/tool_lib/zon_avg/grids/{1}_grid_info.nc'.format(env['DIAGROOTPATH'],env['RESOLUTION']) os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) if env['VERTICAL'] == '42': ## os.environ['gridfile'] = '{0}/tool_lib/zon_avg/grids/{1}_42lev_grid_info.nc'.format(env['DIAGROOTPATH'],env['RESOLUTION']) ## this file doesn't exist! - not sure if this even works now or not os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read') if not rc: print('model_vs_obs: check_prerequisites could not find gridfile = {0}'.format(os.environ['gridfile'])) raise ocn_diags_bc.PrerequisitesError env['GRIDFILE'] = os.environ['gridfile'] # check the resolution and decide if some plot modules should be turned off if env['RESOLUTION'] == 'tx0.1v2' : env['MVO_PM_VELISOPZ'] = os.environ['MVO_PM_VELISOPZ'] = 'FALSE' env['MVO_PM_KAPPAZ'] = os.environ['MVO_PM_KAPPAZ'] = 'FALSE' # create the global zonal average file used by most of the plotting classes print(' model vs. obs - calling create_za') diagUtilsLib.create_za( env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env) return env
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format(self.__class__.__name__)) super(modelTimeseries, self).check_prerequisites(env) # chdir into the working directory os.chdir(env['WORKDIR']) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') cesmEnvLib.purge(env['WORKDIR'], '.*\.log\.*') cesmEnvLib.purge(env['WORKDIR'], '.*\.pop\.d.\.*') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # set the OBSROOT env['OBSROOT'] = env['OBSROOTPATH'] # check the resolution and decide if some plot modules should be turned off if env['RESOLUTION'] == 'tx0.1v2' : env['MTS_PM_MOCANN'] = os.environ['PM_MOCANN'] = 'FALSE' env['MTS_PM_MOCMON'] = os.environ['PM_MOCMON'] = 'FALSE' # check if cpl log file path is defined if len(env['CPLLOGFILEPATH']) == 0: # print a message that the cpl log path isn't defined and turn off CPLLOG plot module print('model timeseries - CPLLOGFILEPATH is undefined. Disabling MTS_PM_CPLLOG module') env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE' else: # check that cpl log files exist and gunzip them if necessary initcplLogs = cplLogs = list() initCplLogs = glob.glob('{0}/cpl.log.*'.format(env['CPLLOGFILEPATH'])) if len(initCplLogs) > 0: for cplLog in initCplLogs: logFileList = cplLog.split('/') cplLogFile = logFileList[-1] shutil.copy2(cplLog, '{0}/{1}'.format(env['WORKDIR'],cplLogFile)) # gunzip the cplLog in the workdir if cplLogFile.lower().find('.gz') != -1: cplLog_gunzip = cplLogFile[:-3] inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],cplLogFile), 'rb') outFile = open('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip), 'wb') outFile.write( inFile.read() ) inFile.close() outFile.close() # append the gunzipped cpl log file to the cplLogs list cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLog_gunzip)) # remove the original .gz file in the workdir os.remove('{0}/{1}'.format(env['WORKDIR'],cplLogFile)) else: # append the original gunzipped cpl log file to the cplLogs list cplLogs.append('{0}/{1}'.format(env['WORKDIR'],cplLogFile)) # parse the cpllog depending on the coupler version - default to 7b print('model_timeseries: setting up heat and freshwater awk calls with cplLogs = {0}'.format(cplLogs)) heatFile = 'cplheatbudget' freshWaterFile = 'cplfwbudget' cplVersion = 'cpl7b' env['ntailht'] = os.environ['ntailht'] = '22' env['ntailfw'] = os.environ['ntailfw'] = '16' if '7' == env['TS_CPL'] or '6' == env['TS_CPL']: cplVersion = 'cpl{0}'.format(env['TS_CPL']) env['ntailht'] = os.environ['ntailht'] = '21' env['ntailfw'] = os.environ['ntailfw'] = '16' # expand the cpl.log* into a list cplLogsString = ' '.join(cplLogs) # define the awk scripts to parse the cpllog file heatPath = '{0}/process_{1}_logfiles_heat.awk'.format(env['TOOLPATH'], cplVersion) heatPath = os.path.abspath(heatPath) fwPath = '{0}/process_{1}_logfiles_fw.awk'.format(env['TOOLPATH'], cplVersion) fwPath = os.path.abspath(fwPath) heatCmd = '{0} y0={1} y1={2} {3}'.format(heatPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ') freshWaterCmd = '{0} y0={1} y1={2} {3}'.format(fwPath, env['TSERIES_YEAR0'], env['TSERIES_YEAR1'], cplLogsString).split(' ') # run the awk scripts to generate the .txt files from the cpllogs cmdList = [ (heatCmd, heatFile, env['ntailht']), (freshWaterCmd, freshWaterFile, env['ntailfw']) ] for cmd in cmdList: outFile = '{0}.txt'.format(cmd[1]) with open (outFile, 'w') as results: try: subprocess.check_call(cmd[0], stdout=results, env=env) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) rc, err_msg = cesmEnvLib.checkFile(outFile, 'read') if rc: # get the tail of the .txt file and redirect to a .asc file for the web ascFile = '{0}.asc'.format(cmd[1]) with open (ascFile, 'w') as results: try: # TODO - read the .txt in and write just the lines needed to avoid subprocess call tailCmd = 'tail -{0} {1}.txt'.format(cmd[2], cmd[1]).split(' ') subprocess.check_call(tailCmd, stdout=results, env=env) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model timeseries - Coupler logs do not exist. Disabling MTS_PM_CPLLOG module') env['MTS_PM_CPLLOG'] = os.environ['PM_CPLLOG'] = 'FALSE' # check if ocn log files exist if len(env['OCNLOGFILEPATH']) == 0: # print a message that the ocn log path isn't defined and turn off POPLOG plot module print('model timeseries - OCNLOGFILEPATH is undefined. Disabling MTS_PM_YPOPLOG module') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' else: # check that ocn log files exist and gunzip them if necessary initOcnLogs = ocnLogs = list() initOcnLogs = glob.glob('{0}/ocn.log.*'.format(env['OCNLOGFILEPATH'])) if len(initOcnLogs) > 0: for ocnLog in initOcnLogs: logFileList = ocnLog.split('/') ocnLogFile = logFileList[-1] shutil.copy2(ocnLog, '{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) # gunzip the ocnLog in the workdir if ocnLogFile.lower().find('.gz') != -1: ocnLog_gunzip = ocnLogFile[:-3] inFile = gzip.open('{0}/{1}'.format(env['WORKDIR'],ocnLogFile), 'rb') outFile = open('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip), 'wb') outFile.write( inFile.read() ) inFile.close() outFile.close() # append the gunzipped ocn log file to the ocnLogs list ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLog_gunzip)) # remove the original .gz file in the workdir os.remove('{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) else: # append the original gunzipped ocn log file to the ocnLogs list ocnLogs.append('{0}/{1}'.format(env['WORKDIR'],ocnLogFile)) # expand the ocn.log* into a list ocnLogs.sort() ocnLogsString = ' '.join(ocnLogs) # define the awk script to parse the ocn log files globalDiagAwkPath = '{0}/process_pop2_logfiles.globaldiag.awk'.format(env['TOOLPATH']) globalDiagAwkCmd = '{0} {1}'.format(globalDiagAwkPath, ocnLogsString).split(' ') print('model_timeseries: globalDiagAwkCmd = {0}'.format(globalDiagAwkCmd)) # run the awk scripts to generate the .txt files from the ocn logs try: subprocess.check_call(globalDiagAwkCmd) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model timeseries - Ocean logs do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLTmodules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' # check if dt files exist if len(env['DTFILEPATH']) == 0: # print a message that the dt file path isn't defined and turn off POPLOG plot module print('model timeseries - DTFILEPATH is undefined. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLTmodules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' else: # check that dt files exist dtFiles = list() dtFiles = glob.glob('{0}/{1}.pop.dt.*'.format(env['DTFILEPATH'], env['CASE'])) print('dtFiles = {0}'.format(dtFiles)) if len(dtFiles) > 0: for dtFile in dtFiles: logFileList = dtFile.split('/') dtLogFile = logFileList[-1] shutil.copy2(dtFile, '{0}/{1}'.format(env['WORKDIR'],dtLogFile)) # expand the *.dt.* into a list dtFiles.sort() dtFilesString = ' '.join(dtFiles) # define the awk script to parse the dt log files dtFilesAwkPath = '{0}/process_pop2_dtfiles.awk'.format(env['TOOLPATH']) dtFilesAwkCmd = '{0} {1}'.format(dtFilesAwkPath, dtFilesString).split(' ') print('model_timeseries: dtFilesAwkCmd = {0}'.format(dtFilesAwkCmd)) # run the awk scripts to generate the .txt files from the dt log files try: subprocess.check_call(dtFilesAwkCmd) except subprocess.CalledProcessError as e: print('WARNING: {0} time series error executing command:'.format(self._name)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('model_timeseries - ocean dt files do not exist. Disabling MTS_PM_YPOPLOG and MTS_PM_ENSOWVLT modules') env['MTS_PM_YPOPLOG'] = os.environ['PM_YPOPLOG'] = 'FALSE' env['MTS_PM_ENSOWVLT'] = os.environ['PM_ENSOWVLT'] = 'FALSE' return env
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format( self.__class__.__name__)) super(modelVsObsEcosys, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T', 'TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # read in the ecosys_vars.txt file and create the corresponding link files to the climatology files try: ecosys_vars_file = env['ECOSYSVARSFILE'] f = open(ecosys_vars_file) ecosys_vars = f.read().split() f.close() except IOError as e: print('ERROR: unable to open {0} error {1} : {2}'.format( ecosys_vars_file, e.errno, e.strerror)) except ValueError: print('ERROR: unable to split {0} into separate variable names'. format(ecosys_vars_file)) except: print('ERROR: unexpected error in {0}'.format(self._name)) raise # loop over the ecosys_vars list and create the links to the mavg file sourceFile = os.path.join( env['WORKDIR'], 'mavg.{0:04d}.{1:04d}.nc'.format(int(env['YEAR0']), int(env['YEAR1']))) for var in ecosys_vars: linkFile = os.path.join( env['WORKDIR'], '{0}.{1}.clim.{2:04d}-{3:04d}.nc'.format( env['CASE'], var, int(env['YEAR0']), int(env['YEAR1']))) try: os.symlink(sourceFile, linkFile) except OSError as e: print( 'ERROR: unable to create symbolic link {0} to {1} error {2} : {3}' .format(linkFile, sourceFile, e.errno, e.strerror)) # create the POPDIAG and PME environment variables env['POPDIAGPY'] = env['POPDIAGPY2'] = env['POPDIAG'] = os.environ[ 'POPDIAG'] = 'TRUE' env['PME'] = os.environ['PME'] = '1' env['mappdir'] = env['ECODATADIR'] + '/mapping' # create the plot_depths.dat file fh = open('{0}/plot_depths.dat'.format(env['WORKDIR']), 'w') fh.write('{0}\n'.format(env['DEPTHS'])) fh.close() return env
def check_prerequisites(self, env): """ check prerequisites """ print(' Checking prerequisites for : {0}'.format(self.__class__.__name__)) self._name = '{0}_{1}'.format(self._name, env['CNTRLCASE']) super(modelVsControl, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # setup prerequisites for the model # setup the gridfile based on the resolution and levels os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) if env['VERTICAL'] == '42': os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) if env['VERTICAL'] == '62': os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read') if not rc: raise OSError(err_msg) env['GRIDFILE'] = os.environ['gridfile'] # check the resolution and decide if some plot modules should be turned off if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3') : env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE' env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE' # create the global zonal average file used by most of the plotting classes print(' model vs. control - calling create_za') diagUtilsLib.create_za( env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env) # setup prerequisites for the model control control = True env['CNTRL_MAVGFILE'], env['CNTRL_TAVGFILE'] = diagUtilsLib.createLinks(env['CNTRLYEAR0'], env['CNTRLYEAR1'], env['CNTRLTAVGDIR'], env['WORKDIR'], env['CNTRLCASE'], control) env['CNTRLFILE'] = env['CNTRL_TAVGFILE'] # setup the gridfile based on the resolution and vertical levels os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION']) if env['VERTICAL'] == '42': os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION']) if env['VERTICAL'] == '62': os.environ['gridfilecntrl'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['CNTRLRESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfilecntrl'], 'read') if not rc: raise OSError(err_msg) env['GRIDFILECNTRL'] = os.environ['gridfilecntrl'] # check the resolution and decide if some plot modules should be turned off if (env['CNTRLRESOLUTION'] == 'tx0.1v2' or env['CNTRLRESOLUTION'] == 'tx0.1v3') : env['MVC_PM_VELISOPZ'] = os.environ['MVC_PM_VELISOPZ'] = 'FALSE' env['MVC_PM_KAPPAZ'] = os.environ['MVC_PM_KAPPAZ'] = 'FALSE' # create the control global zonal average file used by most of the plotting classes print(' model vs. control - calling create_za for control run') diagUtilsLib.create_za( env['WORKDIR'], env['CNTRL_TAVGFILE'], env['GRIDFILECNTRL'], env['TOOLPATH'], env) return env
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format(self.__class__.__name__)) super(modelVsObs, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # setup the gridfile based on the resolution and vertical levels os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) if env['VERTICAL'] == '42': os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_42lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) if env['VERTICAL'] == '62': os.environ['gridfile'] = '{0}/omwg/za_grids/{1}_62lev_grid_info.nc'.format(env['DIAGOBSROOT'],env['RESOLUTION']) # check if gridfile exists and is readable rc, err_msg = cesmEnvLib.checkFile(os.environ['gridfile'], 'read') if not rc: print('model_vs_obs: check_prerequisites could not find gridfile = {0}'.format(os.environ['gridfile'])) raise ocn_diags_bc.PrerequisitesError env['GRIDFILE'] = os.environ['gridfile'] # check the resolution and decide if some plot modules should be turned off if (env['RESOLUTION'] == 'tx0.1v2' or env['RESOLUTION'] == 'tx0.1v3'): env['MVO_PM_VELISOPZ'] = os.environ['MVO_PM_VELISOPZ'] = 'FALSE' env['MVO_PM_KAPPAZ'] = os.environ['MVO_PM_KAPPAZ'] = 'FALSE' # create the global zonal average file used by most of the plotting classes print(' model vs. obs - calling create_za') diagUtilsLib.create_za( env['WORKDIR'], env['TAVGFILE'], env['GRIDFILE'], env['TOOLPATH'], env) return env
def check_prerequisites(self, env): """ check prerequisites """ print(" Checking prerequisites for : {0}".format(self.__class__.__name__)) super(modelVsObsEcosys, self).check_prerequisites(env) # clean out the old working plot files from the workdir if env['CLEANUP_FILES'].upper() in ['T','TRUE']: cesmEnvLib.purge(env['WORKDIR'], '.*\.pro') cesmEnvLib.purge(env['WORKDIR'], '.*\.gif') cesmEnvLib.purge(env['WORKDIR'], '.*\.dat') cesmEnvLib.purge(env['WORKDIR'], '.*\.ps') cesmEnvLib.purge(env['WORKDIR'], '.*\.png') cesmEnvLib.purge(env['WORKDIR'], '.*\.html') # create the plot.dat file in the workdir used by all NCL plotting routines diagUtilsLib.create_plot_dat(env['WORKDIR'], env['XYRANGE'], env['DEPTHS']) # read in the ecosys_vars.txt file and create the corresponding link files to the climatology files try: ecosys_vars_file = env['ECOSYSVARSFILE'] f = open(ecosys_vars_file) ecosys_vars = f.read().split() f.close() except IOError as e: print ('ERROR: unable to open {0} error {1} : {2}'.format(ecosys_vars_file, e.errno, e.strerror)) except ValueError: print ('ERROR: unable to split {0} into separate variable names'.format(ecosys_vars_file)) except: print ('ERROR: unexpected error in {0}'.format(self._name)) raise # loop over the ecosys_vars list and create the links to the mavg file sourceFile = os.path.join(env['WORKDIR'],'mavg.{0:04d}.{1:04d}.nc'.format(int(env['YEAR0']),int(env['YEAR1']))) for var in ecosys_vars: linkFile = os.path.join(env['WORKDIR'],'{0}.{1}.clim.{2:04d}-{3:04d}.nc'.format(env['CASE'],var,int(env['YEAR0']),int(env['YEAR1']))) try: os.symlink(sourceFile, linkFile) except OSError as e: print ('ERROR: unable to create symbolic link {0} to {1} error {2} : {3}'.format(linkFile, sourceFile, e.errno, e.strerror)) # create the POPDIAG and PME environment variables env['POPDIAGPY'] = env['POPDIAGPY2'] = env['POPDIAG'] = os.environ['POPDIAG'] = 'TRUE' env['PME'] = os.environ['PME'] = '1' env['mappdir'] = env['ECODATADIR']+'/mapping' # create the plot_depths.dat file fh = open('{0}/plot_depths.dat'.format(env['WORKDIR']),'w') fh.write('{0}\n'.format(env['DEPTHS'])) fh.close() return env