def testWeightBalanced(self):
results = [set([0, 1, 2]), set([1]), set(),
set([3, 2, 4, 1, 0]), set([1]), set(),
set([3, 2, 4, 1, 5, 0, 6]), set([3, 6]), set([4])]
for (ii, inp) in enumerate(self.inputs):
weights = [(3 - i) ** 2 for i in inp[0]]
pfunc = partition.WeightBalanced()
actual = set(pfunc(list(zip(inp[0], weights)), inp[1], inp[2]))
expected = results[ii]
msg = test_info_msg(
'WeightBalanced', zip(inp[0], weights), inp[1], inp[2], actual, expected)
print(msg)
self.assertEqual(actual, expected, msg)
def testWeightBalanced(self):
results = [
set([0, 1, 2]),
set([1]),
set(),
set([3, 2, 4, 1, 0]),
set([1]),
set(),
set([3, 2, 4, 1, 5, 0, 6]),
set([3, 6]),
set([4]),
]
for (ii, inp) in enumerate(self.inputs):
weights = array([(3 - i)**2 for i in inp[0]])
pfunc = partition.WeightBalanced()
data = dstack((inp[0], weights))[0]
actual = set(pfunc(data, inp[1], inp[2]))
expected = results[ii]
self.assertEqual(actual, expected)
def testWeightBalanced(self):
results = [
set([0, 1, 2]),
set([1]),
set(),
set([3, 2, 4, 1, 0]),
set([1]),
set(),
set([3, 2, 4, 1, 5, 0, 6]),
set([3, 6]),
set([4])
]
for (ii, inp) in enumerate(self.inputs):
weights = numpy.array([(3 - i)**2 for i in inp[0]])
pfunc = partition.WeightBalanced()
data = numpy.dstack((inp[0], weights))[0]
actual = set(pfunc(data, inp[1], inp[2]))
expected = results[ii]
msg = test_info_msg('WeightBalanced', data, inp[1], inp[2], actual,
expected)
print msg
self.assertEqual(actual, expected, msg)
def run_diagnostics(self, env, scomm):
""" call the necessary plotting routines to generate diagnostics plots
"""
super(modelVsModel, self).run_diagnostics(env, scomm)
scomm.sync()
# setup some global variables
requested_plot_sets = list()
local_requested_plots = list()
local_html_list = list()
# all the plot module XML vars start with 'set_' need to strip that off
for key, value in env.iteritems():
if ("wset_" in key
and (value == 'True' or env['all_waccm_sets'] == 'True')):
requested_plot_sets.append(key)
elif ("cset_" in key
and (value == 'True' or env['all_chem_sets'] == 'True')):
requested_plot_sets.append(key)
elif ("set_" in key
and (value == 'True' or env['all_sets'] == 'True')):
if ("wset_" not in key and "cset_" not in key):
requested_plot_sets.append(key)
scomm.sync()
# partition requested plots to all tasks
# first, create plotting classes and get the number of plots each will created
requested_plots = {}
set_sizes = {}
plots_weights = []
for plot_set in requested_plot_sets:
requested_plots.update(
atm_diags_plot_factory.atmosphereDiagnosticPlotFactory(
plot_set, env))
for plot_id, plot_class in requested_plots.iteritems():
if hasattr(plot_class, 'weight'):
factor = plot_class.weight
else:
factor = 1
plots_weights.append(
(plot_id, len(plot_class.expectedPlots) * factor))
# partition based on the number of plots each set will create
local_plot_list = scomm.partition(plots_weights,
func=partition.WeightBalanced(),
involved=True)
timer = timekeeper.TimeKeeper()
# loop over local plot lists - set env and then run plotting script
#for plot_id,plot_class in local_plot_list.interitems():
timer.start(str(scomm.get_rank()) + "ncl total time on task")
for plot_set in local_plot_list:
timer.start(str(scomm.get_rank()) + plot_set)
plot_class = requested_plots[plot_set]
# set all env variables (global and particular to this plot call
plot_class.check_prerequisites(env)
# Stringify the env dictionary
for name, value in plot_class.plot_env.iteritems():
plot_class.plot_env[name] = str(value)
# call script to create plots
for script in plot_class.ncl_scripts:
diagUtilsLib.generate_ncl_plots(plot_class.plot_env, script)
plot_class.plot_env['NCDF_MODE'] = 'write'
plot_class.plot_env['VAR_MODE'] = 'write'
timer.stop(str(scomm.get_rank()) + plot_set)
timer.stop(str(scomm.get_rank()) + "ncl total time on task")
scomm.sync()
print(timer.get_all_times())
w = 0
for p in plots_weights:
if p[0] in local_plot_list:
w = w + p[1]
print(str(scomm.get_rank()) + ' weight:' + str(w))
# set html files
if scomm.is_manager():
env['HTML_HOME'] = env['DIAG_HOME'] + '/html/model1-model2/'
# Get web dir name and create it if it does not exist
#web_dir = '{0}/{1}-{2}'.format(env['test_path_diag'], env['test_casename'], env['cntl_casename'])
web_dir = env['test_path_diag']
#if not os.path.exists(web_dir):
# os.makedirs(web_dir)
# Copy over some files needed by web pages
if not os.path.exists(web_dir + '/images'):
os.mkdir(web_dir + '/images')
diag_imgs = glob.glob(env['DIAG_HOME'] + '/html/images/*')
for img in diag_imgs:
shutil.copy(img, web_dir + '/images/')
# Create set dirs, copy plots to set dir, and create html file for set
requested_plot_sets.append(
'sets') # Add 'sets' to create top level html files
for plot_set in requested_plot_sets:
if 'set_5' == plot_set or 'set_6' == plot_set:
glob_set = plot_set.replace('_', '')
plot_set = 'set5_6'
elif 'set_1' == plot_set:
glob_set = 'table_'
plot_set = plot_set.replace('_', '')
elif 'sets' == plot_set:
set_dir = web_dir + '/'
else:
plot_set = plot_set.replace('_', '')
glob_set = plot_set
if 'sets' not in plot_set: #'sets' is top level, don't create directory or copy images files
set_dir = web_dir + '/' + plot_set
# Create the plot set web directory
if not os.path.exists(set_dir):
os.makedirs(set_dir)
# Copy plots into the correct web dir
glob_string = env['test_path_diag'] + '/' + glob_set + '*.*'
imgs = glob.glob(glob_string)
if imgs > 0:
for img in imgs:
new_fn = set_dir + '/' + os.path.basename(img)
os.rename(img, new_fn)
# Copy/Process html files
if 'sets' in plot_set:
orig_html = env['HTML_HOME'] + '/' + plot_set
else:
orig_html = env[
'HTML_HOME'] + '/' + plot_set + '/' + plot_set
create_atm_html.create_plotset_html(orig_html, set_dir,
plot_set, env,
'model_vs_model')
# Remove any plotvar netcdf files that exists in the diag directory
if env['save_ncdfs'] == 'False':
cesmEnvLib.purge(env['test_path_diag'], '.*\.nc')
cesmEnvLib.purge(env['test_path_diag'], '/station_ids')
# move all the plots to the diag_path with the years appended to the path
endYr1 = (int(env['test_first_yr']) + int(env['test_nyrs'])) - 1
endYr2 = (int(env['cntl_first_yr']) + int(env['cntl_nyrs'])) - 1
diag_path = '{0}/diag/{1}.{2}_{3}-{4}.{5}_{6}'.format(
env['OUTPUT_ROOT_PATH'], env['test_casename'],
env['test_first_yr'], str(endYr1), env['cntl_casename'],
env['cntl_first_yr'], str(endYr2))
move_files = True
try:
os.makedirs(diag_path)
except OSError as exception:
if exception.errno != errno.EEXIST:
err_msg = 'ERROR: {0} problem accessing directory {1}'.format(
self.__class__.__name__, diag_path)
raise OSError(err_msg)
move_files = False
elif env['CLEANUP_FILES'].lower() in ['t', 'true']:
# delete all the files in the diag_path directory
for root, dirs, files in os.walk('diag_path'):
for f in files:
os.unlink(os.path.join(root, f))
for d in dirs:
shutil.rmtree(os.path.join(root, d))
elif env['CLEANUP_FILES'].lower() in ['f', 'false']:
print(
'WARNING: {0} exists and is not empty and ATMDIAG_CLEANUP_FILES = False. Leaving new diagnostics files in {1}'
.format(diag_path, web_dir))
diag_path = web_dir
move_files = False
# move the files to the new diag_path
if move_files:
try:
print('DEBUG: model_vs_model renaming web files')
os.rename(web_dir, diag_path)
except OSError as e:
print('WARNING: Error renaming %s to %s: %s' %
(web_dir, diag_path, e))
diag_path = web_dir
# setup the unique ATMDIAG_WEBDIR_MODEL_VS_OBS output file
env_file = '{0}/env_diags_atm.xml'.format(env['PP_CASE_PATH'])
key = 'ATMDIAG_WEBDIR_{0}'.format(self._name)
value = diag_path
##web_file = '{0}/web_dirs/{1}.{2}-{3}'.format(env['PP_CASE_PATH'], key, scomm.get_size(), scomm.get_rank() )
web_file = '{0}/web_dirs/{1}.{2}'.format(
env['PP_CASE_PATH'], key,
datetime.datetime.now().strftime('%Y-%m-%d_%H%M%S'))
try:
diagUtilsLib.write_web_file(web_file, 'atm', key, value)
except:
print(
'WARNING atm model_vs_model unable to write {0}={1} to {2}'
.format(key, value, web_file))
print(
'*******************************************************************************'
)
print(
'Successfully completed generating atmosphere diagnostics model vs. model plots'
)
print(
'*******************************************************************************'
)