def test_get_all_times(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.05 ok_diff = 0.01 tk.start(name1) sleep(2 * wait_time) tk.start(name2) sleep(wait_time) tk.stop(name1) sleep(wait_time) tk.stop(name2) all_times = tk.get_all_times() expected_all_times = {name1: 3 * wait_time, name2: 2 * wait_time} self.assertListEqual(expected_all_times.keys(), all_times.keys(), 'All times clock names are off') self.assertAlmostEqual(expected_all_times.values()[0], all_times.values()[0], msg='Accumulated time 1 seems off', delta=ok_diff) self.assertAlmostEqual(expected_all_times.values()[1], all_times.values()[1], msg='Accumulated time 2 seems off', delta=ok_diff)
def test_get_all_times(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.05 tk.start(name1) sleep(2 * wait_time) tk.start(name2) sleep(wait_time) tk.stop(name1) sleep(wait_time) tk.stop(name2) all_times = tk.get_all_times() expected_all_times = {name1: 3 * wait_time, name2: 2 * wait_time} self.assertTrue( len(expected_all_times.keys()) == len(all_times.keys())) self.assertTrue( all([ i1 == i2 for i1, i2 in zip(expected_all_times.keys(), all_times.keys()) ])) self.assertAlmostEqual(list(expected_all_times.values())[0], list(all_times.values())[0], places=1) self.assertAlmostEqual(list(expected_all_times.values())[1], list(all_times.values())[1], places=1)
def test_get_all_times(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.05 tk.start(name1) sleep(2 * wait_time) tk.start(name2) sleep(wait_time) tk.stop(name1) sleep(wait_time) tk.stop(name2) all_times = tk.get_all_times() expected_all_times = {name1: 3 * wait_time, name2: 2 * wait_time} self.assertTrue(len(expected_all_times.keys()) == len(all_times.keys()), 'Different number of time clonk names') self.assertTrue(all([i1==i2 for i1,i2 in zip(expected_all_times.keys(),all_times.keys())]), 'Clock names are not the same') self.assertAlmostEqual(list(expected_all_times.values())[0], list(all_times.values())[0], msg='Accumulated time 1 seems off', places=1) self.assertAlmostEqual(list(expected_all_times.values())[1], list(all_times.values())[1], msg='Accumulated time 2 seems off', places=1)
def test_init(self): tk = timekeeper.TimeKeeper() self.assertEqual( type(tk), timekeeper.TimeKeeper, 'TimeKeeper class instantiated incorrectly.', )
def test_reset_values(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) tk.reset(name) self.assertEqual(0, tk.get_time(name))
def test_start_stop_names(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) self.assertTrue(name in tk._accumulated_times) self.assertTrue(name in tk._added_order) self.assertTrue(name in tk._start_times)
def test_get_time(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) dt = tk.get_time(name) dterr = abs(dt / wait_time - 1.0) self.assertTrue(dterr < 0.15, msg='Get time seems off')
def test_start_stop_values(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) dt = tk.get_time(name) dterr = abs(dt / wait_time - 1.0) self.assertTrue(dterr < 0.15)
def test_get_time(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) self.assertAlmostEqual(wait_time, tk.get_time(name), msg='Get time seems off', places=2)
def test_start_stop_values(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 ok_diff = 0.01 tk.start(name) sleep(wait_time) tk.stop(name) self.assertAlmostEqual(wait_time, tk.get_time(name), msg='Accumulated time seems off', delta=ok_diff)
def test_start_stop_names(self): tk = timekeeper.TimeKeeper() name = 'Test Clock' wait_time = 0.05 tk.start(name) sleep(wait_time) tk.stop(name) self.assertTrue(name in tk._accumulated_times, 'Clock name not found in accumulated times dictionary') self.assertTrue(name in tk._added_order, 'Clock name not found in added order list') self.assertTrue(name in tk._start_times, 'Clock name not found in start times dictionary')
def test_start_stop_order_names(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.01 tk.start(name1) sleep(wait_time) tk.stop(name1) tk.start(name2) sleep(wait_time) tk.stop(name2) self.assertEqual(name1, tk._added_order[0]) self.assertEqual(name2, tk._added_order[1])
def test_start_stop_values2(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.05 tk.start(name1) sleep(2 * wait_time) tk.start(name2) sleep(wait_time) tk.stop(name1) sleep(wait_time) tk.stop(name2) dt1 = tk.get_time(name1) dt1err = abs(dt1 / (3 * wait_time) - 1.0) self.assertTrue(dt1err < 0.15, msg='Accumulated time 1 seems off') dt2 = tk.get_time(name2) dt2err = abs(dt2 / (2 * wait_time) - 1.0) self.assertTrue(dt2err < 0.15, msg='Accumulated time 1 seems off')
def test_start_stop_values2(self): tk = timekeeper.TimeKeeper() name1 = 'Test Clock 1' name2 = 'Test Clock 2' wait_time = 0.05 tk.start(name1) sleep(2 * wait_time) tk.start(name2) sleep(wait_time) tk.stop(name1) sleep(wait_time) tk.stop(name2) self.assertAlmostEqual(3 * wait_time, tk.get_time(name1), msg='Accumulated time 1 seems off', places=2) self.assertAlmostEqual(2 * wait_time, tk.get_time(name2), msg='Accumulated time 2 seems off', places=2)
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' need to strip that off for key, value in env.iteritems(): if ("PLOT_" in key and value == 'True'): ## if ("DIFF" in key or 'PLOT_REGIONS' in key): if ("DIFF" in key): requested_plot_sets.append(key) scomm.sync() # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} for plot_set in requested_plot_sets: requested_plots.update( ice_diags_plot_factory.iceDiagnosticPlotFactory(plot_set, env)) # partition based on the number of plots each set will create local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True) timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script #for plot_id,plot_class in local_plot_list.interitems(): timer.start(str(scomm.get_rank()) + "ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank()) + plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) timer.stop(str(scomm.get_rank()) + plot_set) timer.stop(str(scomm.get_rank()) + "ncl total time on task") scomm.sync() print(timer.get_all_times()) # set html files if scomm.is_manager(): # Setup (local) web directories env['HTML_HOME'] = env['POSTPROCESS_PATH'] + '/ice_diag/web/' web_dir = env['WKDIR'] + '/yrs' + env['BEGYR_CONT'] + '-' + env[ 'ENDYR_CONT'] if env['CLEANUP_FILES'].lower() in ['t', 'true' ] and os.path.exists(web_dir): shutil.rmtree(web_dir) if not os.path.exists(web_dir): os.mkdir(web_dir) if not os.path.exists(web_dir + '/contour'): os.mkdir(web_dir + '/contour') if not os.path.exists(web_dir + '/vector'): os.mkdir(web_dir + '/vector') if not os.path.exists(web_dir + '/line'): os.mkdir(web_dir + '/line') if not os.path.exists(web_dir + '/obs'): os.mkdir(web_dir + '/obs') # Move images tot the oppropriate directories plot_dir_map = { 'icesat': 'obs', 'ASPeCt': 'obs', 'diff_con_': 'contour', 'diff_vec_': 'vector', 'line': 'line', 'clim': 'line' } for key, dir in plot_dir_map.iteritems(): glob_string = env['WKDIR'] + '/*' + key + '*.png' imgs = glob.glob(glob_string) if imgs > 0: for img in imgs: new_fn = web_dir + '/' + dir + '/' + os.path.basename( img) os.rename(img, new_fn) # Create/format the html files and copy txt and map files shutil.copy(env['HTML_HOME'] + '/ICESat.txt', web_dir + '/obs/ICESat.txt') shutil.copy(env['HTML_HOME'] + '/ASPeCt.txt', web_dir + '/obs/ASPeCt.txt') glob_string = env['HTML_HOME'] + '/maps/*' maps = glob.glob(glob_string) for map in maps: mn = os.path.basename(map) shutil.copy(map, web_dir + '/' + mn) print('DEBUG: model_vs_model web_dir = {0}'.format(web_dir)) html_dir = env['HTML_HOME'] create_ice_html.create_plotset_html( html_dir + '/index_diff_temp.html', web_dir + '/index.html', env) create_ice_html.create_plotset_html( html_dir + '/contour_diff.html', web_dir + '/contour.html', env) create_ice_html.create_plotset_html( html_dir + '/timeseries_diff.html', web_dir + '/timeseries.html', env) create_ice_html.create_plotset_html( html_dir + '/regional_diff.html', web_dir + '/regional.html', env) create_ice_html.create_plotset_html(html_dir + '/vector_diff.html', web_dir + '/vector.html', env) # setup the unique ICEDIAG_WEBDIR_MODEL_VS_MODEL output file env_file = '{0}/env_diags_ice.xml'.format(env['PP_CASE_PATH']) key = 'ICEDIAG_WEBDIR_{0}'.format(self._name) value = env['WKDIR'] ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format( env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'ice', key, value) except: print( 'WARNING ice model_vs_model unable to write {0}={1} to {2}' .format(key, value, web_file)) print( '*******************************************************************************' ) print( 'Successfully completed generating ice diagnostics model vs. model plots' ) print( '*******************************************************************************' )
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' need to strip that off for key, value in env.iteritems(): if ("wset_" in key and (value == 'True' or env['all_waccm_sets'] == 'True')): requested_plot_sets.append(key) elif ("cset_" in key and (value == 'True' or env['all_chem_sets'] == 'True')): requested_plot_sets.append(key) elif ("set_" in key and (value == 'True' or env['all_sets'] == 'True')): if ("wset_" not in key and "cset_" not in key): requested_plot_sets.append(key) scomm.sync() # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update( atm_diags_plot_factory.atmosphereDiagnosticPlotFactory( plot_set, env)) for plot_id, plot_class in requested_plots.iteritems(): if hasattr(plot_class, 'weight'): factor = plot_class.weight else: factor = 1 plots_weights.append( (plot_id, len(plot_class.expectedPlots) * factor)) # partition based on the number of plots each set will create local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script #for plot_id,plot_class in local_plot_list.interitems(): timer.start(str(scomm.get_rank()) + "ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank()) + plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) plot_class.plot_env['NCDF_MODE'] = 'write' plot_class.plot_env['VAR_MODE'] = 'write' timer.stop(str(scomm.get_rank()) + plot_set) timer.stop(str(scomm.get_rank()) + "ncl total time on task") scomm.sync() print(timer.get_all_times()) w = 0 for p in plots_weights: if p[0] in local_plot_list: w = w + p[1] print(str(scomm.get_rank()) + ' weight:' + str(w)) # set html files if scomm.is_manager(): env['HTML_HOME'] = env['DIAG_HOME'] + '/html/model1-model2/' # Get web dir name and create it if it does not exist #web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename']) web_dir = env['test_path_diag'] #if not os.path.exists(web_dir): # os.makedirs(web_dir) # Copy over some files needed by web pages if not os.path.exists(web_dir + '/images'): os.mkdir(web_dir + '/images') diag_imgs = glob.glob(env['DIAG_HOME'] + '/html/images/*') for img in diag_imgs: shutil.copy(img, web_dir + '/images/') # Create set dirs, copy plots to set dir, and create html file for set requested_plot_sets.append( 'sets') # Add 'sets' to create top level html files for plot_set in requested_plot_sets: if 'set_5' == plot_set or 'set_6' == plot_set: glob_set = plot_set.replace('_', '') plot_set = 'set5_6' elif 'set_1' == plot_set: glob_set = 'table_' plot_set = plot_set.replace('_', '') elif 'sets' == plot_set: set_dir = web_dir + '/' else: plot_set = plot_set.replace('_', '') glob_set = plot_set if 'sets' not in plot_set: #'sets' is top level, don't create directory or copy images files set_dir = web_dir + '/' + plot_set # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir glob_string = env['test_path_diag'] + '/' + glob_set + '*.*' imgs = glob.glob(glob_string) if imgs > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) # Copy/Process html files if 'sets' in plot_set: orig_html = env['HTML_HOME'] + '/' + plot_set else: orig_html = env[ 'HTML_HOME'] + '/' + plot_set + '/' + plot_set create_atm_html.create_plotset_html(orig_html, set_dir, plot_set, env, 'model_vs_model') # Remove any plotvar netcdf files that exists in the diag directory if env['save_ncdfs'] == 'False': cesmEnvLib.purge(env['test_path_diag'], '.*\.nc') cesmEnvLib.purge(env['test_path_diag'], '/station_ids') # move all the plots to the diag_path with the years appended to the path endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1 endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1 diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format( env['OUTPUT_ROOT_PATH'], env['test_casename'], env['test_first_yr'], str(endYr1), env['cntl_casename'], env['cntl_first_yr'], str(endYr2)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format( self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t', 'true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk('diag_path'): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f', 'false']: print( 'WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}' .format(diag_path, web_dir)) diag_path = web_dir move_files = False # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_model renaming web files') os.rename(web_dir, diag_path) except OSError as e: print('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir # setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH']) key = 'ATMDIAG_WEBDIR_{0}'.format(self._name) value = diag_path ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format( env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'atm', key, value) except: print( 'WARNING atm model_vs_model unable to write {0}={1} to {2}' .format(key, value, web_file)) print( '*******************************************************************************' ) print( 'Successfully completed generating atmosphere diagnostics model vs. model plots' ) print( '*******************************************************************************' )
def compute_averages(self, spec): ''' Sets up the run information and computes the averages. @param spec An instance of the Specification class which holds the user settings that define which averages to compute, directories, file prefixes, etc ''' import os, sys import rover import climAverager import climFileIO import average_types as ave_t import regionOpts import string import collections from asaptools import timekeeper from asaptools import partition #============================================================================== # # Initialize # #============================================================================== # Initialize the timekeeper class and start 'total' timer timer = timekeeper.TimeKeeper() timer.start("Total Time") # Initialize some timers that are not used by all tasks timer.reset("Send Average Time") timer.reset("Variable fetch time") timer.reset("Recv Average Time") timer.reset("Write Netcdf Averages") timer.reset("Variable fetch time") timer.reset("Time to compute Average") # Check average list to make sure it complies with the standards ave_t.average_compliance(spec.avg_list) # Check if I'm the global master g_master = spec.main_comm.is_manager() for tag in spec.m_id: file_pattern = list(spec.file_pattern) if ('-999' not in tag): prefix = spec.prefix + '_' + tag p_index = file_pattern.index('$prefix') t_index = file_pattern.index('$m_id') for i in range(p_index + 1, t_index + 1): del file_pattern[p_index + 1] else: prefix = spec.prefix # Sort through the average list and figure out dependencies and do # averages in steps if need be. avg_dict = {0: spec.avg_list} for i in range(1, 20): avg_dict[i] = [] avg_dict = ave_t.sort_depend(avg_dict, 0, spec.out_directory, prefix, spec.regions) print avg_dict # Initialize the tag for the average send/recv AVE_TAG = 40 VNAME_TAG = 41 #start_level = 0 start_level = min(avg_dict.keys()) found_level = False #for i in range(0,len(avg_dict)): # if found_level == False: # if (i in avg_dict): # start_level = i # found_level = True ordered_avg_dict = collections.OrderedDict(avg_dict) #for i in range(start_level,len(avg_dict)): for i, value in ordered_avg_dict.items(): # Initialize some containers var_list = [] full_hist_dict = {} hist_dict = {} #============================================================================== # # Set the hist_dict up with file references for all years/months. # Create a list of all variables and meta variables within the file # and set the final variable list passed on user preferences. # #============================================================================== ## Set var_list and file info dictionary timer.start("Define history dictionary") if (spec.hist_type == 'series'): full_hist_dict, full_var_list, meta_list, key = rover.set_slices_and_vars_time_series( spec.in_directory, file_pattern, spec.date_pattern, prefix, spec.suffix, spec.year0, spec.year1, spec.split, spec.split_files) else: full_hist_dict, full_var_list, meta_list, key = rover.set_slices_and_vars_time_slice( spec.in_directory, file_pattern, prefix, spec.suffix, spec.year0, spec.year1) timer.stop("Define history dictionary") # Set variable list. If there was a variable list passed to the averager, use this list. Other wise, # use all variables within the file. if (len(spec.varlist) > 0): var_list = spec.varlist for v in full_var_list: if '__meta' in v: var_list.append(v) else: var_list = full_var_list meta_list = list(set(meta_list)) var_list = list(set(var_list)) #============================================================================== # # Workload Distribution # #============================================================================== # Each intercommunicator recieves a list of averages it's responsible for # Each mpi task within that intercommunicator gets a portion of the variable list num_of_avg = len(avg_dict[i]) min_procs_per_ave = min(4, spec.main_comm.get_size()) # Override user selection if they picked less than 2 or # the variable list is less than the min procs per sub-communicator if (min_procs_per_ave < 2 or len(var_list) <= (min_procs_per_ave - 1)): min_procs_per_ave = 2 # If running in paralllel mode, split the communicator and partition the averages if (spec.serial == False): size = spec.main_comm.get_size() rank = spec.main_comm.get_rank() # split mpi comm world temp_color = (rank // min_procs_per_ave) % num_of_avg num_of_groups = size / min_procs_per_ave if (temp_color == num_of_groups): temp_color = temp_color - 1 groups = [] for g in range(0, num_of_groups): groups.append(g) #print 'g_rank:',rank,'size:',size,'#of ave:',num_of_avg,'min_procs:',min_procs_per_ave,'temp_color:',temp_color,'#of groups',num_of_groups,'groups:',groups group = groups[temp_color] inter_comm, multi_comm = spec.main_comm.divide(group) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() #g_master = spec.main_comm.is_manager() l_master = inter_comm.is_manager() #print 'global rank: ',rank,'local rank: ',lrank,'color: ',color,'tempcolor: ',temp_color,'group: ',group,'is local master: ',l_master laverages = [] AVE_LIST_TAG = 50 # Partion the average task list amoung the inter/split communicators if (l_master): laverages = multi_comm.partition( avg_dict[i], func=partition.EqualStride(), involved=True) for b in range(1, lsize): laverages_send = inter_comm.ration( data=laverages, tag=AVE_LIST_TAG) else: laverages = inter_comm.ration(tag=AVE_LIST_TAG) else: # Running in serial mode. Just copy the average list. laverages = avg_dict[i] inter_comm = spec.main_comm lsize = inter_comm.get_size() #g_master = spec.main_comm.is_manager() l_master = inter_comm.is_manager() # Partition the variable list between the tasks of each communicator if (lsize > 1 and spec.serial == False): lvar_list = inter_comm.partition( var_list, func=partition.EqualStride(), involved=False) if (l_master): lvar_list = var_list else: lvar_list = var_list #print rank,lvar_list #print(rank,'averages :',laverages, ' vars :',lvar_list) #============================================================================== # # Create the output directory if it doesn't exist # #============================================================================== if spec.serial or g_master: if not os.path.exists(spec.out_directory): os.makedirs(spec.out_directory) spec.main_comm.sync() #============================================================================== # # Main Averaging Loop # #============================================================================== # Files are only split for the first loop. When the depend averages start, they will operate on files # that are already stiched together. if (i != 0): spec.split_name = 'null' spec.split = False spec.split_files = 'null' # Toggle to incate that extra variables were added to the local file list (only do once per average level added_extra_vars = False for ave in laverages: for split_name in spec.split_files.split(","): # Split apart the average info to get type of average and year(s) ave_descr = ave.split(':') if ('hor.meanyr' in ave_descr[0] or 'hor.meanConcat' in ave_descr[0]): ave_name_split = ave_descr[0].split('_') region_num = ave_name_split[len(ave_name_split) - 1] region_name = spec.regions[int(region_num)] # Remove the region number as part of the average name ave_descr[0] = ave_name_split[0] else: region_name = 'null' region_num = -99 # If the average depends on other averages that have to be computed, create a new temporary dictionary if '__d' in ave_descr: yr0 = ave_descr[1] if (len(ave_descr) > 2 and '_d' not in ave_descr[2]): yr1 = ave_descr[2] else: yr1 = ave_descr[1] hist_dict = rover.set_slices_and_vars_depend( spec.out_directory, file_pattern, prefix, yr0, yr1, ave_t.average_types[ave_descr[0]], ave_descr[0], region_name) else: hist_dict = dict(full_hist_dict) # If concat' mean_diff_rms files, for each var, also add the _DIFF and _RMS variables. if ('hor.meanConcat' in ave_descr and added_extra_vars == False): new_vars = [] for v in lvar_list: if '__meta' not in v: new_vars.append(v + '_DIFF') new_vars.append(v + '_RMS') lvar_list = lvar_list + new_vars added_extra_vars = True # Create and define the average file timer.start("Create/Define Netcdf File") if (len(ave_descr) < 3 or 'hor.meanyr' in ave_descr): ave_date = string.zfill(ave_descr[1], 4) ave_date2 = str(ave_descr[1]) else: date1 = string.zfill(ave_descr[1], 4) date2 = string.zfill(ave_descr[2], 4) ave_date = date1 + '-' + date2 ave_date2 = str(ave_descr[1]) + '-' + str( ave_descr[2]) outfile_name = climFileIO.get_out_fn( ave_descr[0], prefix, ave_date, ave_t.average_types[ave_descr[0]]['fn'], region_name) if 'zonalavg' in ave_descr: l_collapse_dim = spec.collapse_dim else: l_collapse_dim = '' all_files_vars, new_file = climFileIO.define_ave_file( l_master, spec.serial, var_list, lvar_list, meta_list, hist_dict, spec.hist_type, ave_descr, prefix, outfile_name, spec.split, split_name, spec.out_directory, inter_comm, spec.ncformat, ave_t.average_types[ ave_descr[0]]['months_to_average'][0], key, spec.clobber, spec.year0, spec.year1, ave_date2, collapse_dim=l_collapse_dim) timer.stop("Create/Define Netcdf File") # Start loops to compute averages # create a list of years that are needed for this average years = [] if '__d' in ave_descr: if (ave_t.average_types[ ave_descr[0]]['depend_type'] == 'month' or '_d' in ave_descr[2]): years.append(int(ave_descr[1])) else: years = list( range(int(ave_descr[1]), int(ave_descr[2]) + 1)) depend = True else: if (len(ave_descr) == 2): years.append(int(ave_descr[1])) else: years = list( range(int(ave_descr[1]), int(ave_descr[2]) + 1)) depend = False # Get the first year. If part of a sig avg, this will be the sig first year, not year of indiv average fyr = years[0] if i + 1 in avg_dict.keys(): for a in avg_dict[i + 1]: if (ave_descr[0] + '_sig') in a: spl = a.split(':') fyr = int(spl[1]) file_dict = [] open_list = [] # Open all of the files that this rank will need for this average (for time slice files) if ((spec.hist_type == 'slice' or '__d' in ave_descr) and (spec.serial or not l_master) and len(lvar_list) > 0): file_dict = [] open_list = [] file_dict, open_list = climFileIO.open_all_files( hist_dict, ave_t.average_types[ave_descr[0]] ['months_to_average'], years, lvar_list[0], 'null', ave_descr[0], depend, fyr) # If concat of file instead of average, piece file together here. If not, enter averaging loop if (('mavg' in ave_descr or 'moc' in ave_descr or 'annall' in ave_descr or 'mons' in ave_descr or '_mean' in ave_descr[0]) and len(lvar_list) > 0): file_dict = [] open_list = [] if (spec.serial or not l_master): # Open files file_dict, open_list = climFileIO.open_all_files( hist_dict, ave_t.average_types[ ave_descr[0]]['months_to_average'], years, lvar_list[0], 'null', ave_descr[0], depend, fyr) # Loop through variables and compute the averages for orig_var in lvar_list: # Some variable names were suffixed with a meta label indicaticating that the variable exists in all files, # but there isn't a didicated ts file to open. Pick the first variable off the list and get values from there if ('__meta' in orig_var): var = key else: var = orig_var # Open all of the files that this rank will need for this average (for time series files) if ((spec.hist_type == 'series' and '__d' not in ave_descr) and (spec.serial or not l_master)): if ('mavg' not in ave_descr or 'moc' not in ave_descr or 'annall' not in ave_descr or 'mons' not in ave_descr or '_mean' not in ave_descr[0]): file_dict = [] open_list = [] file_dict, open_list = climFileIO.open_all_files( hist_dict, ave_t.average_types[ ave_descr[0]]['months_to_average'], years, var, split_name, ave_descr[0], depend, fyr) # We now have open files to pull values from. Now reset var name if ('__meta' in orig_var): parts = orig_var.split('__') var = parts[0] # If concat, all of the procs will participate in this call if ('mavg' in ave_descr or 'moc' in ave_descr or 'mocm' in ave_descr or 'hor.meanConcat' in ave_descr or 'annall' in ave_descr or 'mons' in ave_descr or '_mean' in ave_descr[0] or 'zonalavg' in ave_descr): if 'zonalavg' in ave_descr: l_collapse_dim = spec.collapse_dim else: l_collapse_dim = '' # Concat var_avg_results = climAverager.time_concat( var, years, hist_dict, ave_t.average_types[ave_descr[0]], file_dict, ave_descr[0], inter_comm, all_files_vars, spec.serial, timer, collapse_dim=spec.collapse_dim) # Else (not concat), each slave will compute averages and each master will collect and write else: if spec.serial or not l_master: # mean_diff_rsm file if ('hor.meanyr' in ave_descr and '__meta' not in orig_var): obs_file = spec.obs_dir + "/" + spec.obs_file reg_obs_file = spec.obs_dir + "/" + region_name + spec.reg_obs_file_suffix # The mean diff rsm function will send the variables once they are created var_avg_results, var_DIFF_results, var_RMS_results = climAverager.mean_diff_rms( var, region_name, region_num, spec.region_nc_var, spec.region_wgt_var, years, hist_dict, ave_t.average_types[ave_descr[0]], file_dict, obs_file, reg_obs_file, inter_comm, spec.serial, VNAME_TAG, AVE_TAG) else: if ('__metaChar' in orig_var): # Handle special meta var_avg_results = climAverager.get_metaCharValue( var, years, hist_dict, ave_t.average_types[ ave_descr[0]], file_dict, timer) else: # Average if (spec.weighted == True and 'weights' in ave_t.average_types[ ave_descr[0]]): var_avg_results = climAverager.weighted_avg_var( var, years, hist_dict, ave_t.average_types[ ave_descr[0]], file_dict, ave_descr[0], timer, depend, fyr) else: var_avg_results = climAverager.avg_var( var, years, hist_dict, ave_t.average_types[ ave_descr[0]], file_dict, ave_descr[0], timer, depend, fyr) # Close all open files (for time series files) if ((spec.hist_type == 'series' and '__d' not in ave_descr) and (spec.serial or not l_master)): climFileIO.close_all_files( open_list) # Pass the average results to master rank for writing var_shape = var_avg_results.shape var_dtype = var_avg_results.dtype var_type = type(var_avg_results) md_message = { 'name': var, 'shape': var_shape, 'dtype': var_dtype, 'average': var_avg_results, 'type': var_type } if not spec.serial: timer.start("Send Average Time") #inter_comm.collect(data=md_message, tag=AVE_TAG) inter_comm.collect(data=var, tag=VNAME_TAG) inter_comm.collect( data=var_avg_results, tag=AVE_TAG) timer.stop("Send Average Time") if spec.serial or l_master: # If ave_descr is hor.meanyr, there will be three variables to write for each variable. # Other wise, there will only be 1 if ('hor.meanyr' in ave_descr and '__meta' not in orig_var): var_cnt = 3 else: var_cnt = 1 for r in range(0, var_cnt): if not spec.serial: timer.start("Recv Average Time") #r_rank,results = inter_comm.collect(tag=AVE_TAG) #r_var_avg_results = results['average'] r_rank, var_name = inter_comm.collect( tag=VNAME_TAG) r_rank, r_var_avg_results = inter_comm.collect( tag=AVE_TAG) #var_name = results['name'] timer.start("Recv Average Time") else: var_name = var r_var_avg_results = var_avg_results timer.start("Write Netcdf Averages") climFileIO.write_averages( all_files_vars, r_var_avg_results, var_name) if ('hor.meanyr' in ave_descr and spec.serial ) and '__meta' not in orig_var: climFileIO.write_averages( all_files_vars, var_DIFF_results, var_name + '_DIFF') climFileIO.write_averages( all_files_vars, var_RMS_results, var_name + '_RMS') timer.stop("Write Netcdf Averages") # Close all open files (for time slice files) if (('mavg' in ave_descr or 'moc__d' == ave_descr[0] or 'annall' in ave_descr or 'mons' in ave_descr or '_mean' in ave_descr[0]) and len(lvar_list) > 0): if (spec.serial or not l_master): climFileIO.close_all_files(open_list) elif ((spec.hist_type == 'slice' or '__d' in ave_descr) and (spec.serial or not l_master) and len(lvar_list) > 0): climFileIO.close_all_files(open_list) # Sync the local communicator before closing the averaged netcdf file and moving to the next average inter_comm.sync() # Close the newly created average file if spec.serial or l_master: new_file.close() # If needed, stitch spatially split files together. if spec.serial or l_master: if (len(spec.split_files.split(",")) > 1): fn1 = spec.out_directory + '/nh_' + outfile_name fn2 = spec.out_directory + '/sh_' + outfile_name out_fn = spec.out_directory + '/' + outfile_name dim_info = spec.split_orig_size.split(",") dim1 = dim_info[0].split("=") dim2 = dim_info[1].split("=") regionOpts.combine_regions(fn1, fn2, out_fn, dim1[0], int(dim1[1]), dim2[0], int(dim2[1]), "nj", spec.clobber) if not spec.serial: # Free the inter-communicators #intercomm.Free() # Sync all mpi tasks / All averages should have been computed at this point spec.main_comm.sync() #============================================================================== # # Collect and print timing information # #============================================================================== timer.stop("Total Time") my_times = spec.main_comm.allreduce(timer.get_all_times(), 'max') if g_master: print("==============================================") print "COMPLETED SUCCESSFULLY" print my_times print("==============================================")
def run_diagnostics(self, env, scomm): """ call the necessary plotting routines to generate diagnostics plots """ super(modelVsModel, self).run_diagnostics(env, scomm) scomm.sync() # setup some global variables requested_plot_sets = list() local_requested_plots = list() local_html_list = list() # all the plot module XML vars start with 'set_' for key, value in env.iteritems(): if ("set_" in key and value == 'True'): requested_plot_sets.append(key) scomm.sync() if scomm.is_manager(): print('DEBUG model_vs_model requested_plot_sets = {0}'.format( requested_plot_sets)) # partition requested plots to all tasks # first, create plotting classes and get the number of plots each will created requested_plots = {} set_sizes = {} #plots_weights = [] for plot_set in requested_plot_sets: requested_plots.update( lnd_diags_plot_factory.LandDiagnosticPlotFactory( plot_set, env)) #for plot_id,plot_class in requested_plots.iteritems(): # if hasattr(plot_class, 'weight'): # factor = plot_class.weight # else: # factor = 1 # plots_weights.append((plot_id,len(plot_class.expectedPlots)*factor)) # partition based on the number of plots each set will create #local_plot_list = scomm.partition(plots_weights, func=partition.WeightBalanced(), involved=True) if scomm.is_manager(): print('DEBUG model_vs_model requested_plots.keys = {0}'.format( requested_plots.keys())) local_plot_list = scomm.partition(requested_plots.keys(), func=partition.EqualStride(), involved=True) scomm.sync() timer = timekeeper.TimeKeeper() # loop over local plot lists - set env and then run plotting script timer.start(str(scomm.get_rank()) + "ncl total time on task") for plot_set in local_plot_list: timer.start(str(scomm.get_rank()) + plot_set) plot_class = requested_plots[plot_set] # set all env variables (global and particular to this plot call print( 'model vs. model - Checking prerequisite for {0} on rank {1}'. format(plot_class.__class__.__name__, scomm.get_rank())) plot_class.check_prerequisites(env) # Stringify the env dictionary for name, value in plot_class.plot_env.iteritems(): plot_class.plot_env[name] = str(value) # call script to create plots for script in plot_class.ncl_scripts: print( 'model vs. model - Generating plots for {0} on rank {1} with script {2}' .format(plot_class.__class__.__name__, scomm.get_rank(), script)) diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script) timer.stop(str(scomm.get_rank()) + plot_set) timer.stop(str(scomm.get_rank()) + "ncl total time on task") scomm.sync() print(timer.get_all_times()) #w = 0 #for p in plots_weights: # if p[0] in local_plot_list: # w = w + p[1] #print(str(scomm.get_rank())+' weight:'+str(w)) # set html files if scomm.is_manager(): # Create web dirs and move images/tables to that web dir for n in ('1', '2', '3', '4', '5', '6', '7', '8', '9'): web_dir = env['WKDIR'] set_dir = web_dir + '/set' + n # Create the plot set web directory if not os.path.exists(set_dir): os.makedirs(set_dir) # Copy plots into the correct web dir glob_string = web_dir + '/set' + n + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) # Copy the set1Diff and set1Anom plots to set_1 web dir if n == '1': glob_string = web_dir + '/set1Diff' + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) glob_string = web_dir + '/set1Anom' + '_*' imgs = glob.glob(glob_string) if len(imgs) > 0: for img in imgs: new_fn = set_dir + '/' + os.path.basename(img) os.rename(img, new_fn) env['WEB_DIR'] = web_dir shutil.copy2( env['POSTPROCESS_PATH'] + '/lnd_diag/inputFiles/' + env['VAR_MASTER'], web_dir + '/variable_master.ncl') if n == '9': web_script = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_statTable.pl' rc2, err_msg = cesmEnvLib.checkFile(web_script, 'read') if rc2: try: subprocess.check_call(web_script) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script)) # copy the set9_statTable.html to the set9 subdir set9_statTable = web_dir + '/set9_statTable.html' rc2, err_msg = cesmEnvLib.checkFile(set9_statTable, 'read') if rc2: new_fn = web_dir + '/set9/' + os.path.basename( set9_statTable) try: os.rename(set9_statTable, new_fn) except os.OSError as e: print('WARNING: rename error for file {0}'.format( set9_statTable)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, set9_statTable)) web_script_1 = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_create_webpage.pl' web_script_2 = env[ 'POSTPROCESS_PATH'] + '/lnd_diag/shared/lnd_lookupTable.pl' print('Creating Web Pages') # set the shell environment cesmEnvLib.setXmlEnv(env) # lnd_create_webpage.pl call rc1, err_msg = cesmEnvLib.checkFile(web_script_1, 'read') if rc1: try: subprocess.check_call(web_script_1) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script_1)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script_1)) # lnd_lookupTable.pl call rc2, err_msg = cesmEnvLib.checkFile(web_script_2, 'read') if rc2: try: subprocess.check_call(web_script_2) except subprocess.CalledProcessError as e: print('WARNING: {0} error executing command:'.format( web_script_2)) print(' {0}'.format(e.cmd)) print(' rc = {0}'.format(e.returncode)) else: print('{0}... {1} file not found'.format( err_msg, web_script_2)) # move all the plots to the diag_path with the years appended to the path endYr1 = (int(env['clim_first_yr_1']) + int(env['clim_num_yrs_1'])) - 1 endYr2 = (int(env['clim_first_yr_2']) + int(env['clim_num_yrs_2'])) - 1 diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format( env['OUTPUT_ROOT_PATH'], env['caseid_1'], env['clim_first_yr_1'], str(endYr1), env['caseid_2'], env['clim_first_yr_2'], str(endYr2)) move_files = True try: os.makedirs(diag_path) except OSError as exception: if exception.errno != errno.EEXIST: err_msg = 'ERROR: {0} problem accessing directory {1}'.format( self.__class__.__name__, diag_path) raise OSError(err_msg) move_files = False elif env['CLEANUP_FILES'].lower() in ['t', 'true']: # delete all the files in the diag_path directory for root, dirs, files in os.walk('diag_path'): for f in files: os.unlink(os.path.join(root, f)) for d in dirs: shutil.rmtree(os.path.join(root, d)) elif env['CLEANUP_FILES'].lower() in ['f', 'false']: print( 'WARNING: {0} exists and is not empty and LNDDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}' .format(diag_path, web_dir)) diag_path = web_dir move_files = False # move the files to the new diag_path if move_files: try: print('DEBUG: model_vs_model renaming web files') os.rename(web_dir, diag_path) except OSError as e: print('WARNING: Error renaming %s to %s: %s' % (web_dir, diag_path, e)) diag_path = web_dir print('DEBUG: model vs. model web_dir = {0}'.format(web_dir)) print('DEBUG: model vs. model diag_path = {0}'.format(diag_path)) # setup the unique LNDDIAG_WEBDIR_MODEL_VS_MODEL output file env_file = '{0}/env_diags_lnd.xml'.format(env['PP_CASE_PATH']) key = 'LNDDIAG_WEBDIR_{0}'.format(self._name) value = diag_path ##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() ) web_file = '{0}/web_dirs/{1}.{2}'.format( env['PP_CASE_PATH'], key, datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S')) try: diagUtilsLib.write_web_file(web_file, 'lnd', key, value) except: print( 'WARNING lnd model_vs_model unable to write {0}={1} to {2}' .format(key, value, web_file)) print( '*******************************************************************************' ) print( 'Successfully completed generating land diagnostics model vs. model plots' ) print( '*******************************************************************************' )
def main(options, scomm, rank, size): """ """ # setup an overall timer timer = timekeeper.TimeKeeper() timer.start("Total Time") # initialize the CASEROOT environment dictionary cesmEnv = dict() # CASEROOT is given on the command line as required option --caseroot caseroot = options.caseroot[0] # set the debug level debug = options.debug[0] # is there only one mip definition in each file? ind = "True" # get the XML variables loaded into a hash env_file_list = ['env_postprocess.xml','env_conform.xml'] cesmEnv = cesmEnvLib.readXML(caseroot, env_file_list); # We want to have warnings and not errors (at least for the first sets of cmip simulations) simplefilter("default", ValidationWarning) # Get the extra modules pyconform needs pp_path = cesmEnv["POSTPROCESS_PATH"] conform_module_path = pp_path+'/conformer/conformer/source/pyconform/modules/' for i, m in enumerate(external_mods): print("Loading: "+conform_module_path+"/"+m) load_source('user{}'.format(i), conform_module_path+"/"+m) # create the cesm stream to table mapping # if rank == 0: dout_s_root = cesmEnv['TIMESERIES_OUTPUT_ROOTDIR'] case = cesmEnv['CASE'] pc_inpur_dir = cesmEnv['CONFORM_JSON_DIRECTORY']+'/PyConform_input/' #readArchiveXML(caseroot, dout_s_root, case, debug) nc_files = find_nc_files(dout_s_root) variable_list = fill_list(nc_files, pc_inpur_dir, cesmEnv["CONFORM_EXTRA_FIELD_NETCDF_DIR"], scomm, rank, size) mappings = {} if rank == 0: mappings = match_tableSpec_to_stream(pc_inpur_dir, variable_list, dout_s_root, case) for k,v in sorted(mappings.iteritems()): print k for f in sorted(v): print f print len(v),'\n\n' scomm.sync() # Pass the stream and mapping information to the other procs mappings = scomm.partition(mappings, func=partition.Duplicate(), involved=True) print("I CAN RUN ",len(mappings.keys())," json files") failures = 0 if len(mappings.keys()) > 0: # setup subcommunicators to do streams and chunks in parallel # everyone participates except for root inter_comm, lsubcomms = divide_comm(scomm, len(mappings.keys()), ind) color = inter_comm.get_color() lsize = inter_comm.get_size() lrank = inter_comm.get_rank() print "MPI INFO: ",color," ",lrank,"/",lsize," ",rank,"/",size GWORK_TAG = 10 # global comm mpi tag LWORK_TAG = 20 # local comm mpi tag # global root - hands out mappings to work on. When complete, it must tell each subcomm all work is done. if (rank == 0): #for i in range(0,len(mappings.keys())): # hand out all mappings for i in mappings.keys(): scomm.ration(data=i, tag=GWORK_TAG) for i in range(1,lsubcomms): # complete, signal this to all subcomms scomm.ration(data=-99, tag=GWORK_TAG) # subcomm root - performs the same tasks as other subcomm ranks, but also gets the specifier to work on and sends # this information to all ranks within subcomm elif (lrank == 0): i = -999 while i != -99: i = scomm.ration(tag=GWORK_TAG) # recv from global for x in range(1,lsize): inter_comm.ration(i, LWORK_TAG) # send to local ranks if i != -99: print "(",rank,"/",lrank,")"," start running ",i failures += run_PyConform(i, mappings[i], inter_comm) print "(",rank,"/",lrank,")"," finished running ",i print "(",rank,"/",lrank,")","FAILURES: ",failures inter_comm.sync() # all subcomm ranks - recv the specifier to work on and call the reshaper else: i = -999 while i != -99: i = inter_comm.ration(tag=LWORK_TAG) # recv from local root if i != -99: print "(",rank,"/",lrank,")"," start running ",i failures += run_PyConform(i, mappings[i], inter_comm) print "(",rank,"/",lrank,")"," finished running ",i print "(",rank,"/",lrank,")","FAILURES: ",failures inter_comm.sync() print "(",rank,"/",lrank,")"," FINISHED" scomm.sync() timer.stop("Total Time") if rank == 0: print('************************************************************') print('Total Time: {0} seconds'.format(timer.get_time("Total Time"))) print('************************************************************')
chunking.write_log('{0}/logs/ts_status.log'.format(caseroot), log) debugMsg('after chunking.write_log', header=True, verbosity=1) scomm.sync() return 0 #=================================== if __name__ == "__main__": # initialize simplecomm object scomm = simplecomm.create_comm(serial=False) # setup an overall timer timer = timekeeper.TimeKeeper() timer.start("Total Time") # get commandline options options = commandline_options() debug = options.debug[0] # initialize global vprinter object for printing debug messages debugMsg = vprinter.VPrinter(header='', verbosity=0) if options.debug: header = 'cesm_tseries_generator: DEBUG... ' debugMsg = vprinter.VPrinter(header=header, verbosity=options.debug[0]) rank = scomm.get_rank() size = scomm.get_size()
def test_init(self): tk = timekeeper.TimeKeeper() self.assertIsInstance(tk, timekeeper.TimeKeeper, 'TimeKeeper class instantiated incorrectly.')