def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, 'ref_name', '')
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(parameter, ref_data, season)
# Get land/ocean fraction for masking.
try:
land_frac = test_data.get_climo_variable('LANDFRAC', season)
ocean_frac = test_data.get_climo_variable('OCNFRAC', season)
except:
mask_path = os.path.join(acme_diags.INSTALL_PATH, 'acme_ne30_ocean_land_mask.nc')
with cdms2.open(mask_path) as f:
land_frac = f('LANDFRAC')
ocean_frac = f('OCNFRAC')
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
for region in regions:
print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
mv1_domain_mean = mean(mv1_domain)
mv2_domain_mean = mean(mv2_domain)
diff = mv1_domain_mean - mv2_domain_mean
mv1_domain_mean.id = var
mv2_domain_mean.id = var
diff.id = var
parameter.backend = 'mpl' # For now, there's no vcs support for this set.
plot(parameter.current_set, mv2_domain_mean,
mv1_domain_mean, diff, {}, parameter)
utils.general.save_ncfiles(parameter.current_set,
mv1_domain, mv2_domain, diff, parameter)
return parameter
def run_diag(parameter):
parameter.reference_data_path
parameter.test_data_path
variables = parameter.variables
seasons = parameter.seasons
ref_name = parameter.ref_name
regions = parameter.regions
for season in seasons:
try:
filename1 = utils.get_test_filename(parameter, season)
filename2 = utils.get_ref_filename(parameter, season)
except IOError as e:
print(e)
# the file for the current parameters wasn't found, move to next
# parameters
continue
print('test file: {}'.format(filename1))
print('reference file: {}'.format(filename2))
f_mod = cdms2.open(filename1)
f_obs = cdms2.open(filename2)
if parameter.short_test_name:
parameter.test_name_yrs = parameter.short_test_name
else:
parameter.test_name_yrs = parameter.test_name
try:
yrs_averaged = f_mod.getglobal('yrs_averaged')
parameter.test_name_yrs = parameter.test_name_yrs + ' (' + yrs_averaged + ')'
except:
print('No yrs_averaged exists in global attributes')
parameter.test_name_yrs = parameter.test_name_yrs
# save land/ocean fraction for masking
try:
land_frac = f_mod('LANDFRAC')
ocean_frac = f_mod('OCNFRAC')
except BaseException:
mask_path = os.path.join(acme_diags.INSTALL_PATH,
'acme_ne30_ocean_land_mask.nc')
f0 = cdms2.open(mask_path)
land_frac = f0('LANDFRAC')
ocean_frac = f0('OCNFRAC')
f0.close()
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = acme.process_derived_var(var, acme.derived_variables, f_mod,
parameter)
mv2 = acme.process_derived_var(var, acme.derived_variables, f_obs,
parameter)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
# special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with denise
if ref_name == 'WARREN':
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# following should move to derived variable
if ref_name == 'AIRS':
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 > 1e+20, mv2)
if ref_name == 'WILLMOTT' or ref_name == 'CLOUDSAT':
print(mv2.fill_value)
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -999., mv2)
print(mv2.fill_value)
# following should move to derived variable
if var == 'PRECT_LAND':
days_season = {
'ANN': 365,
'DJF': 90,
'MAM': 92,
'JJA': 92,
'SON': 91
}
# mv1 = mv1 * days_season[season] * 0.1 #following AMWG
# approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm
mv2 = mv2 / days_season[season] / \
0.1 # convert cm to mm/day instead
mv2.units = 'mm/day'
if mv1.getLevel() and mv2.getLevel(): # for variables with z axis:
plev = parameter.plevs
print('Selected pressure level: {}'.format(plev))
f_ins = [f_mod, f_obs]
for f_ind, mv in enumerate([mv1, mv2]):
mv_plv = mv.getLevel()
# var(time,lev,lon,lat) convert from hybrid level to
# pressure
if mv_plv.long_name.lower().find('hybrid') != -1:
f_in = f_ins[f_ind]
hyam = f_in('hyam')
hybm = f_in('hybm')
ps = f_in('PS') # Pa
mv_p = utils.hybrid_to_plevs(mv, hyam, hybm, ps, plev)
# levels are presure levels
elif mv_plv.long_name.lower().find(
'pressure') != -1 or mv_plv.long_name.lower().find(
'isobaric') != -1:
mv_p = utils.pressure_to_plevs(mv, plev)
else:
raise RuntimeError(
"Vertical level is neither hybrid nor pressure. Abort"
)
if f_ind == 0:
mv1_p = mv_p
if f_ind == 1:
mv2_p = mv_p
# select plev
for ilev in range(len(plev)):
mv1 = mv1_p[ilev, ]
mv2 = mv2_p[ilev, ]
# select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print("Selected region: {}".format(region))
mv1_domain, mv2_domain = utils.select_region(
region, mv1, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join([
ref_name, var,
str(int(plev[ilev])), season, region
])
parameter.main_title = str(' '.join(
[var,
str(int(plev[ilev])), 'mb', season, region]))
# Regrid towards lower resolution of two variables for
# calculating difference
mv1_reg, mv2_reg = utils.regrid_to_lower_res(
mv1_domain, mv2_domain, parameter.regrid_tool,
parameter.regrid_method)
# Plotting
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(parameter.current_set, mv2_domain, mv1_domain,
diff, metrics_dict, parameter)
utils.save_ncfiles(parameter.current_set, mv1_domain,
mv2_domain, diff, parameter)
f_in.close()
# for variables without z axis:
elif mv1.getLevel() is None and mv2.getLevel() is None:
# select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print("Selected region: {}".format(region))
mv1_domain, mv2_domain = utils.select_region(
region, mv1, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
# regrid towards lower resolution of two variables for
# calculating difference
mv1_reg, mv2_reg = utils.regrid_to_lower_res(
mv1_domain, mv2_domain, parameter.regrid_tool,
parameter.regrid_method)
# if var is 'SST' or var is 'TREFHT_LAND': #special case
if var == 'TREFHT_LAND' or var == 'SST': # use "==" instead of "is"
if ref_name == 'WILLMOTT':
mv2_reg = MV2.masked_where(
mv2_reg == mv2_reg.fill_value, mv2_reg)
print(ref_name)
# if mv.mask is False:
# mv = MV2.masked_less_equal(mv, mv._FillValue)
# print("*************",mv.count())
land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
mv1_reg = MV2.masked_where(land_mask, mv1_reg)
mv2_reg = MV2.masked_where(land_mask, mv2_reg)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(parameter.current_set, mv2_domain, mv1_domain, diff,
metrics_dict, parameter)
utils.save_ncfiles(parameter.current_set, mv1_domain,
mv2_domain, diff, parameter)
else:
raise RuntimeError(
"Dimensions of two variables are difference. Abort")
f_obs.close()
f_mod.close()
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, 'ref_name', '')
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
# Get land/ocean fraction for masking.
try:
land_frac = test_data.get_climo_variable('LANDFRAC', season)
ocean_frac = test_data.get_climo_variable('OCNFRAC', season)
except:
mask_path = os.path.join(acme_diags.INSTALL_PATH,
'acme_ne30_ocean_land_mask.nc')
with cdms2.open(mask_path) as f:
land_frac = f('LANDFRAC')
ocean_frac = f('OCNFRAC')
for var in variables:
print('Variable: {}'.format(var))
#test = test_data.get_timeseries_variable(var)
#ref = ref_data.get_timeseries_variable(var)
test = test_data.get_climo_variable(var, season)
ref = ref_data.get_climo_variable(var, season)
parameter.var_id = var
parameter.viewer_descr[var] = test.long_name if hasattr(
test, 'long_name') else 'No long_name attr in test data.'
for region in regions:
#print("Selected region: {}".format(region))
test_domain = utils.general.select_region(
region, test, land_frac, ocean_frac, parameter)
ref_domain = utils.general.select_region(
region, ref, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join(
[var, 'Diurnal Cycle ', season, region]))
test_cmean, test_amplitude, test_maxtime = utils.diurnal_cycle.composite_diurnal_cycle(
test_domain, season)
ref_cmean, ref_amplitude, ref_maxtime = utils.diurnal_cycle.composite_diurnal_cycle(
ref_domain, season)
metrics_dict = {}
parameter.var_region = region
plot(parameter.current_set, test_maxtime, test_amplitude,
ref_maxtime, ref_amplitude, parameter)
utils.general.save_ncfiles(parameter.current_set, test_cmean,
test_amplitude, test_maxtime,
parameter)
utils.general.save_ncfiles(parameter.current_set, ref_cmean,
ref_amplitude, ref_maxtime,
parameter)
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, 'ref_name', '')
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
# Get land/ocean fraction for masking.
try:
land_frac = test_data.get_climo_variable('LANDFRAC', season)
ocean_frac = test_data.get_climo_variable('OCNFRAC', season)
except:
mask_path = os.path.join(acme_diags.INSTALL_PATH,
'acme_ne30_ocean_land_mask.nc')
with cdms2.open(mask_path) as f:
land_frac = f('LANDFRAC')
ocean_frac = f('OCNFRAC')
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
# Special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with Denis.
if ref_name == 'WARREN':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# The following should be moved to a derived variable.
if ref_name == 'AIRS':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 > 1e+20, mv2)
if ref_name == 'WILLMOTT' or ref_name == 'CLOUDSAT':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -999., mv2)
# The following should be moved to a derived variable.
if var == 'PRECT_LAND':
days_season = {
'ANN': 365,
'DJF': 90,
'MAM': 92,
'JJA': 92,
'SON': 91
}
# mv1 = mv1 * days_season[season] * 0.1 # following AMWG
# Approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm.
mv2 = mv2 / days_season[season] / \
0.1 # Convert cm to mm/day instead.
mv2.units = 'mm/day'
# For variables with a z-axis.
if mv1.getLevel() and mv2.getLevel():
plev = parameter.plevs
print('Selected pressure level: {}'.format(plev))
mv1_p = utils.general.convert_to_pressure_levels(
mv1, plev, test_data, var, season)
mv2_p = utils.general.convert_to_pressure_levels(
mv2, plev, ref_data, var, season)
# Select plev.
for ilev in range(len(plev)):
mv1 = mv1_p[ilev, ]
mv2 = mv2_p[ilev, ]
for region in regions:
print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(
region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(
region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join([
ref_name, var,
str(int(plev[ilev])), season, region
])
parameter.main_title = str(' '.join(
[var,
str(int(plev[ilev])), 'mb', season, region]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = utils.general.regrid_to_lower_res(
mv1_domain, mv2_domain, parameter.regrid_tool,
parameter.regrid_method)
# Plotting
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(parameter.current_set, mv2_domain, mv1_domain,
diff, metrics_dict, parameter)
utils.general.save_ncfiles(parameter.current_set,
mv1_domain, mv2_domain,
diff, parameter)
# For variables without a z-axis.
elif mv1.getLevel() is None and mv2.getLevel() is None:
for region in regions:
print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(
region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(
region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = utils.general.regrid_to_lower_res(
mv1_domain, mv2_domain, parameter.regrid_tool,
parameter.regrid_method)
# Special case.
if var == 'TREFHT_LAND' or var == 'SST':
if ref_name == 'WILLMOTT':
mv2_reg = MV2.masked_where(
mv2_reg == mv2_reg.fill_value, mv2_reg)
land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
mv1_reg = MV2.masked_where(land_mask, mv1_reg)
mv2_reg = MV2.masked_where(land_mask, mv2_reg)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(parameter.current_set, mv2_domain, mv1_domain, diff,
metrics_dict, parameter)
utils.general.save_ncfiles(parameter.current_set,
mv1_domain, mv2_domain, diff,
parameter)
else:
raise RuntimeError(
"Dimensions of the two variables are different. Aborting.")
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, "ref_name", "")
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
# Get land/ocean fraction for masking.
try:
land_frac = test_data.get_climo_variable("LANDFRAC", season)
ocean_frac = test_data.get_climo_variable("OCNFRAC", season)
except Exception:
mask_path = os.path.join(acme_diags.INSTALL_PATH,
"acme_ne30_ocean_land_mask.nc")
with cdms2.open(mask_path) as f:
land_frac = f("LANDFRAC")
ocean_frac = f("OCNFRAC")
for var in variables:
print("Variable: {}".format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = (mv1.long_name if hasattr(
mv1, "long_name") else "No long_name attr in test data.")
# Special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with Denis.
if ref_name == "WARREN":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# The following should be moved to a derived variable.
if ref_name == "AIRS":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 > 1e20, mv2)
if ref_name == "WILLMOTT" or ref_name == "CLOUDSAT":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -999.0, mv2)
# The following should be moved to a derived variable.
if var == "PRECT_LAND":
days_season = {
"ANN": 365,
"DJF": 90,
"MAM": 92,
"JJA": 92,
"SON": 91,
}
# mv1 = mv1 * days_season[season] * 0.1 # following AMWG
# Approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm.
mv2 = (mv2 / days_season[season] / 0.1
) # Convert cm to mm/day instead.
mv2.units = "mm/day"
# For variables with a z-axis.
if mv1.getLevel() and mv2.getLevel():
plev = parameter.plevs
print("Selected pressure level: {}".format(plev))
mv1_p = utils.general.convert_to_pressure_levels(
mv1, plev, test_data, var, season)
mv2_p = utils.general.convert_to_pressure_levels(
mv2, plev, ref_data, var, season)
# Select plev.
for ilev in range(len(plev)):
mv1 = mv1_p[ilev, ]
mv2 = mv2_p[ilev, ]
for region in regions:
# print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(
region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(
region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = "-".join([
ref_name,
var,
str(int(plev[ilev])),
season,
region,
])
parameter.main_title = str(" ".join([
var,
str(int(plev[ilev])),
"mb",
season,
region,
]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = utils.general.regrid_to_lower_res(
mv1_domain,
mv2_domain,
parameter.regrid_tool,
parameter.regrid_method,
)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
# Saving the metrics as a json.
metrics_dict["unit"] = mv1_reg.units
fnm = os.path.join(
utils.general.get_output_dir(
parameter.current_set, parameter),
parameter.output_file + ".json",
)
with open(fnm, "w") as outfile:
json.dump(metrics_dict, outfile)
# Get the filename that the user has passed in and display that.
# When running in a container, the paths are modified.
fnm = os.path.join(
utils.general.get_output_dir(
parameter.current_set,
parameter,
ignore_container=True,
),
parameter.output_file + ".json",
)
# print('Metrics saved in: ' + fnm)
parameter.var_region = region
plot(
parameter.current_set,
mv2_domain,
mv1_domain,
diff,
metrics_dict,
parameter,
)
utils.general.save_ncfiles(
parameter.current_set,
mv1_domain,
mv2_domain,
diff,
parameter,
)
# For variables without a z-axis.
elif mv1.getLevel() is None and mv2.getLevel() is None:
for region in regions:
# print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(
region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(
region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = "-".join(
[ref_name, var, season, region])
parameter.main_title = str(" ".join([var, season, region]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = utils.general.regrid_to_lower_res(
mv1_domain,
mv2_domain,
parameter.regrid_tool,
parameter.regrid_method,
)
# Special case.
if var == "TREFHT_LAND" or var == "SST":
if ref_name == "WILLMOTT":
mv2_reg = MV2.masked_where(
mv2_reg == mv2_reg.fill_value, mv2_reg)
land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
mv1_reg = MV2.masked_where(land_mask, mv1_reg)
mv2_reg = MV2.masked_where(land_mask, mv2_reg)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
# Saving the metrics as a json.
metrics_dict["unit"] = mv1_reg.units
fnm = os.path.join(
utils.general.get_output_dir(parameter.current_set,
parameter),
parameter.output_file + ".json",
)
with open(fnm, "w") as outfile:
json.dump(metrics_dict, outfile)
# Get the filename that the user has passed in and display that.
# When running in a container, the paths are modified.
fnm = os.path.join(
utils.general.get_output_dir(
parameter.current_set,
parameter,
ignore_container=True,
),
parameter.output_file + ".json",
)
# print('Metrics saved in: ' + fnm)
parameter.var_region = region
plot(
parameter.current_set,
mv2_domain,
mv1_domain,
diff,
metrics_dict,
parameter,
)
utils.general.save_ncfiles(
parameter.current_set,
mv1_domain,
mv2_domain,
diff,
parameter,
)
else:
raise RuntimeError(
"Dimensions of the two variables are different. Aborting.")
return parameter
def run_diag(parameter):
reference_data_path = parameter.reference_data_path
test_data_path = parameter.test_data_path
variables = parameter.variables
seasons = parameter.seasons
ref_name = parameter.ref_name
regions = parameter.regions
for season in seasons:
try:
filename1 = utils.get_test_filename(parameter, season)
filename2 = utils.get_ref_filename(parameter, season)
except IOError as e:
print(e)
# the file for the current parameters wasn't found, move to next parameters
continue
print('test file: {}'.format(filename1))
print('reference file: {}'.format(filename2))
f_mod = cdms2.open(filename1)
f_obs = cdms2.open(filename2)
#save land/ocean fraction for masking
try:
land_frac = f_mod('LANDFRAC')
ocean_frac = f_mod('OCNFRAC')
except:
mask_path = os.path.join(sys.prefix, 'share', 'acme_diags',
'acme_ne30_ocean_land_mask.nc')
f0 = cdms2.open(mask_path)
land_frac = f0('LANDFRAC')
ocean_frac = f0('OCNFRAC')
f0.close()
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = acme.process_derived_var(var, acme.derived_variables, f_mod,
parameter)
mv2 = acme.process_derived_var(var, acme.derived_variables, f_obs,
parameter)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
# special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with denise
if ref_name == 'WARREN':
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# following should move to derived variable
if ref_name == 'AIRS':
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
print(mv2.fill_value)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 > 1e+20, mv2)
if ref_name == 'WILLMOTT' or ref_name == 'CLOUDSAT':
print(mv2.fill_value)
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -999., mv2)
print(mv2.fill_value)
# following should move to derived variable
if var == 'PRECT_LAND':
days_season = {
'ANN': 365,
'DJF': 90,
'MAM': 92,
'JJA': 92,
'SON': 91
}
# mv1 = mv1 * days_season[season] * 0.1 #following AMWG
# approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm
mv2 = mv2 / days_season[season] / \
0.1 # convert cm to mm/day instead
mv2.units = 'mm/day'
if mv1.getLevel() and mv2.getLevel(): # for variables with z axis:
#plev = parameter.plevs
plev = numpy.logspace(2.0, 3.0, num=17)
#plev = [30.,50.,70.,100.,150.,200.,250.,300.,400.,500.,600.,700.,775.,850.,925.,1000.]
print('Selected pressure level: {}'.format(plev))
f_ins = [f_mod, f_obs]
for f_ind, mv in enumerate([mv1, mv2]):
mv_plv = mv.getLevel()
# var(time,lev,lon,lat) convert from hybrid level to pressure
if mv_plv.long_name.lower().find('hybrid') != -1:
f_in = f_ins[f_ind]
hyam = f_in('hyam')
hybm = f_in('hybm')
ps = f_in('PS') #Pa
mv_p = utils.hybrid_to_plevs(mv, hyam, hybm, ps, plev)
elif mv_plv.long_name.lower().find(
'pressure') != -1 or mv_plv.long_name.lower().find(
'isobaric') != -1: # levels are presure levels
mv_p = utils.pressure_to_plevs(mv, plev)
else:
raise RuntimeError(
"Vertical level is neither hybrid nor pressure. Abort"
)
#calculate zonal mean
mv_p = cdutil.averager(mv_p, axis='x')
if f_ind == 0:
mv1_p = mv_p
elif f_ind == 1:
mv2_p = mv_p
parameter.output_file = '-'.join(
[ref_name, var, season, parameter.regions[0]])
parameter.main_title = str(' '.join([var, season]))
# Regrid towards lower resolution of two variables for
# calculating difference
if len(mv1_p.getLatitude()) <= len(mv2_p.getLatitude()):
mv1_reg = mv1_p
lev_out = mv1_p.getLevel()
lat_out = mv1_p.getLatitude()
mv2_reg = mv2_p.crossSectionRegrid(lev_out, lat_out)
#apply mask back, since crossSectionRegrid doesn't preserve mask
mv2_reg = MV2.masked_where(mv2_reg == mv2_reg.fill_value,
mv2_reg)
print(mv2_reg.fill_value)
else:
mv2_reg = mv2_p
lev_out = mv2_p.getLevel()
lat_out = mv2_p.getLatitude()
mv1_reg = mv1_p.crossSectionRegrid(lev_out, lat_out)
#apply mask back, since crossSectionRegrid doesn't preserve mask
mv1_reg = MV2.masked_where(mv1_reg == mv1_reg.fill_value,
mv1_reg)
#print(mv1_p.shape, mv2_p.shape)
#mv1_reg, mv2_reg = utils.sregrid_to_lower_res(
# mv1_p, mv2_p, parameter.regrid_tool, parameter.regrid_method)
#reg_mask = MV2.logical_and(mv1_reg.mask, mv2_reg.mask)
#print('reg_mask', reg_mask[:,-1])
#mv1_reg = MV2.masked_where(reg_mask,mv1_reg)
#mv2_reg = MV2.masked_where(reg_mask,mv2_reg)
#print(mv1_reg.shape)
#print(mv2_reg.shape)
#print(mv1_p[:,-1].mask)
#print(mv2_p[:,-1].mask)
#print(mv1_reg[:,-1].mask)
#print(mv2_reg[:,-1].mask)
# Plotting
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_p, mv1_p, mv2_reg, mv1_reg,
diff)
parameter.var_region = 'global'
plot(parameter.current_set, mv2_p, mv1_p, diff, metrics_dict,
parameter)
utils.save_ncfiles(parameter.current_set, mv1_p, mv2_p, diff,
parameter)
# for variables without z axis:
elif mv1.getLevel() == None and mv2.getLevel() == None:
#select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print("Selected region: {}".format(region))
mv1_domain, mv2_domain = utils.select_region(
region, mv1, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
# regrid towards lower resolution of two variables for
# calculating difference
mv1_reg, mv2_reg = utils.regrid_to_lower_res(
mv1_domain, mv2_domain, parameter.regrid_tool,
parameter.regrid_method)
# if var is 'SST' or var is 'TREFHT_LAND': #special case
if var == 'TREFHT_LAND' or var == 'SST': # use "==" instead of "is"
if ref_name == 'WILLMOTT':
mv2_reg = MV2.masked_where(
mv2_reg == mv2_reg.fill_value, mv2_reg)
print(ref_name)
# if mv.mask is False:
# mv = MV2.masked_less_equal(mv, mv._FillValue)
# print("*************",mv.count())
land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
mv1_reg = MV2.masked_where(land_mask, mv1_reg)
mv2_reg = MV2.masked_where(land_mask, mv2_reg)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(parameter.current_set, mv2_domain, mv1_domain, diff,
metrics_dict, parameter)
utils.save_ncfiles(parameter.current_set, mv1_domain,
mv2_domain, diff, parameter)
else:
raise RuntimeError(
"Dimensions of two variables are difference. Abort")
f_obs.close()
f_mod.close()
return parameter
def run_diag(parameter):
reference_data_path = parameter.reference_data_path
test_data_path = parameter.test_data_path
variables = parameter.variables
seasons = parameter.seasons
ref_name = parameter.ref_name
regions = parameter.regions
for season in seasons:
try:
filename1 = utils.get_test_filename(parameter, season)
filename2 = utils.get_ref_filename(parameter, season)
except IOError as e:
print(e)
# the file for the current parameters wasn't found, move to next parameters
continue
print('test file: {}'.format(filename1))
print('reference file: {}'.format(filename2))
f_mod = cdms2.open(filename1)
f_obs = cdms2.open(filename2)
#save land/ocean fraction for masking
try:
land_frac = f_mod('LANDFRAC')
ocean_frac = f_mod('OCNFRAC')
except:
mask_path = os.path.join(sys.prefix, 'share', 'acme_diags',
'acme_ne30_ocean_land_mask.nc')
f0 = cdms2.open(mask_path)
land_frac = f0('LANDFRAC')
ocean_frac = f0('OCNFRAC')
f0.close()
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = acme.process_derived_var(var, acme.derived_variables, f_mod,
parameter)
mv2 = acme.process_derived_var(var, acme.derived_variables, f_obs,
parameter)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
#select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print("Selected region: {}".format(region))
mv1_domain, mv2_domain = utils.select_region(
region, mv1, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
mv1_domain_mean = mean(mv1_domain)
mv2_domain_mean = mean(mv2_domain)
diff = mv1_domain_mean - mv2_domain_mean
mv1_domain_mean.id = var
mv2_domain_mean.id = var
diff.id = var
parameter.backend = 'cartopy'
plot(parameter.current_set, mv2_domain_mean, mv1_domain_mean,
diff, {}, parameter)
# plot(parameter.current_set, mv2_domain, mv1_domain, diff, metrics_dict, parameter)
# utils.save_ncfiles(parameter.current_set, mv1_domain, mv2_domain, diff, parameter)
f_obs.close()
f_mod.close()
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, "ref_name", "")
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
for var in variables:
print("Variable: {}".format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = (mv1.long_name if hasattr(
mv1, "long_name") else "No long_name attr in test data.")
# For variables with a z-axis.
if mv1.getLevel() and mv2.getLevel():
# Since the default is now stored in MeridionalMean2dParameter,
# we must get it from there if the plevs param is blank.
plevs = parameter.plevs
if (isinstance(plevs, numpy.ndarray) and not plevs.all()) or (
not isinstance(plevs, numpy.ndarray) and not plevs):
plevs = ZonalMean2dParameter().plevs
# print('Selected pressure level: {}'.format(plevs))
mv1_p = utils.general.convert_to_pressure_levels(
mv1, plevs, test_data, var, season)
mv2_p = utils.general.convert_to_pressure_levels(
mv2, plevs, ref_data, var, season)
mv1_p = cdutil.averager(mv1_p, axis="y")
mv2_p = cdutil.averager(mv2_p, axis="y")
parameter.output_file = "-".join(
[ref_name, var, season, parameter.regions[0]])
parameter.main_title = str(" ".join([var, season]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
if len(mv1_p.getLongitude()) < len(mv2_p.getLongitude()):
mv1_reg = mv1_p
lev_out = mv1_p.getLevel()
lon_out = mv1_p.getLongitude()
# in order to use regrid tool we need to have at least two latitude bands, so generate new grid first
lat = cdms2.createAxis([0])
lat.setBounds(numpy.array([-1, 1]))
lat.designateLatitude()
grid = cdms2.createRectGrid(lat, lon_out)
data_shape = list(mv2_p.shape)
data_shape.append(1)
mv2_reg = MV2.resize(mv2_p, data_shape)
mv2_reg.setAxis(-1, lat)
for i, ax in enumerate(mv2_p.getAxisList()):
mv2_reg.setAxis(i, ax)
mv2_reg = mv2_reg.regrid(grid, regridTool="regrid2")[...,
0]
# Apply the mask back, since crossSectionRegrid
# doesn't preserve the mask.
mv2_reg = MV2.masked_where(mv2_reg == mv2_reg.fill_value,
mv2_reg)
elif len(mv1_p.getLongitude()) > len(mv2_p.getLongitude()):
mv2_reg = mv2_p
lev_out = mv2_p.getLevel()
lon_out = mv2_p.getLongitude()
mv1_reg = mv1_p.crossSectionRegrid(lev_out, lon_out)
# In order to use regrid tool we need to have at least two
# latitude bands, so generate new grid first.
lat = cdms2.createAxis([0])
lat.setBounds(numpy.array([-1, 1]))
lat.designateLatitude()
grid = cdms2.createRectGrid(lat, lon_out)
data_shape = list(mv1_p.shape)
data_shape.append(1)
mv1_reg = MV2.resize(mv1_p, data_shape)
mv1_reg.setAxis(-1, lat)
for i, ax in enumerate(mv1_p.getAxisList()):
mv1_reg.setAxis(i, ax)
mv1_reg = mv1_reg.regrid(grid, regridTool="regrid2")[...,
0]
# Apply the mask back, since crossSectionRegrid
# doesn't preserve the mask.
mv1_reg = MV2.masked_where(mv1_reg == mv1_reg.fill_value,
mv1_reg)
else:
mv1_reg = mv1_p
mv2_reg = mv2_p
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_p, mv1_p, mv2_reg, mv1_reg,
diff)
parameter.var_region = "global"
plot(
parameter.current_set,
mv2_p,
mv1_p,
diff,
metrics_dict,
parameter,
)
utils.general.save_ncfiles(parameter.current_set, mv1_p, mv2_p,
diff, parameter)
# For variables without a z-axis.
elif mv1.getLevel() is None and mv2.getLevel() is None:
raise RuntimeError(
"One of or both data doesn't have z dimention. Aborting.")
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, 'ref_name', '')
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
for var in variables:
print('Variable: {}'.format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = mv1.long_name if hasattr(
mv1, 'long_name') else 'No long_name attr in test data.'
# Special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with Denis.
if ref_name == 'WARREN':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# The following should be moved to a derived variable.
if ref_name == 'AIRS':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 > 1e+20, mv2)
if ref_name == 'WILLMOTT' or ref_name == 'CLOUDSAT':
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -999., mv2)
# The following should be moved to a derived variable.
if var == 'PRECT_LAND':
days_season = {
'ANN': 365,
'DJF': 90,
'MAM': 92,
'JJA': 92,
'SON': 91
}
# mv1 = mv1 * days_season[season] * 0.1 # following AMWG
# Approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm.
mv2 = mv2 / days_season[season] / \
0.1 # Convert cm to mm/day instead.
mv2.units = 'mm/day'
# For variables with a z-axis.
if mv1.getLevel() and mv2.getLevel():
plev = parameter.plevs
print('Selected pressure level: {}'.format(plev))
mv1_p = utils.general.convert_to_pressure_levels(
mv1, plev, test_data, var, season)
mv2_p = utils.general.convert_to_pressure_levels(
mv2, plev, test_data, var, season)
# Select plev.
for ilev in range(len(plev)):
mv1 = mv1_p[ilev, ]
mv2 = mv2_p[ilev, ]
for region in regions:
#print("Selected region: {}".format(region))
mv1_zonal = cdutil.averager(mv1, axis='x')
mv2_zonal = cdutil.averager(mv2, axis='x')
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = regrid_to_lower_res_1d(
mv1_zonal, mv2_zonal)
diff = mv1_reg - mv2_reg
parameter.output_file = '-'.join([
ref_name, var,
str(int(plev[ilev])), season, region
])
parameter.main_title = str(' '.join(
[var,
str(int(plev[ilev])), 'mb', season, region]))
parameter.var_region = region
plot(parameter.current_set, mv2_zonal, mv1_zonal, diff,
{}, parameter)
utils.general.save_ncfiles(parameter.current_set,
mv1_zonal, mv2_zonal, diff,
parameter)
# For variables without a z-axis.
elif mv1.getLevel() is None and mv2.getLevel() is None:
for region in regions:
#print("Selected region: {}".format(region))
mv1_zonal = cdutil.averager(mv1, axis='x')
mv2_zonal = cdutil.averager(mv2, axis='x')
mv1_reg, mv2_reg = regrid_to_lower_res_1d(
mv1_zonal, mv2_zonal)
diff = mv1_reg - mv2_reg
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
parameter.var_region = region
plot(parameter.current_set, mv2_zonal, mv1_zonal, diff, {},
parameter)
utils.general.save_ncfiles(parameter.current_set,
mv1_zonal, mv2_zonal, diff,
parameter)
else:
raise RuntimeError(
"Dimensions of the two variables are different. Aborting.")
return parameter
def run_diag(parameter):
variables = parameter.variables
seasons = parameter.seasons
ref_name = getattr(parameter, "ref_name", "")
regions = parameter.regions
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
for season in seasons:
# Get the name of the data, appended with the years averaged.
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data, season)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data, season)
# Get land/ocean fraction for masking.
try:
land_frac = test_data.get_climo_variable("LANDFRAC", season)
ocean_frac = test_data.get_climo_variable("OCNFRAC", season)
except Exception:
mask_path = os.path.join(acme_diags.INSTALL_PATH,
"acme_ne30_ocean_land_mask.nc")
with cdms2.open(mask_path) as f:
land_frac = f("LANDFRAC")
ocean_frac = f("OCNFRAC")
for var in variables:
print("Variable: {}".format(var))
parameter.var_id = var
mv1 = test_data.get_climo_variable(var, season)
mv2 = ref_data.get_climo_variable(var, season)
parameter.viewer_descr[var] = (mv1.long_name if hasattr(
mv1, "long_name") else "No long_name attr in test data.")
# Special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with Denis.
if ref_name == "WARREN":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# The following should be moved to a derived variable.
if ref_name == "AIRS":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 > 1e20, mv2)
if ref_name == "WILLMOTT" or ref_name == "CLOUDSAT":
# This is cdms2 fix for bad mask, Denis' fix should fix this.
mv2 = MV2.masked_where(mv2 == -999.0, mv2)
# The following should be moved to a derived variable.
if var == "PRECT_LAND":
days_season = {
"ANN": 365,
"DJF": 90,
"MAM": 92,
"JJA": 92,
"SON": 91,
}
# mv1 = mv1 * days_season[season] * 0.1 # following AMWG
# Approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm.
mv2 = (mv2 / days_season[season] / 0.1
) # Convert cm to mm/day instead.
mv2.units = "mm/day"
# For variables with a z-axis.
if mv1.getLevel() and mv2.getLevel():
# Since the default is now stored in ZonalMean2dParameter,
# we must get it from there if the plevs param is blank.
plevs = parameter.plevs
if (isinstance(plevs, numpy.ndarray) and not plevs.all()) or (
not isinstance(plevs, numpy.ndarray) and not plevs):
plevs = ZonalMean2dParameter().plevs
# print('Selected pressure level: {}'.format(plevs))
mv1_p = utils.general.convert_to_pressure_levels(
mv1, plevs, test_data, var, season)
mv2_p = utils.general.convert_to_pressure_levels(
mv2, plevs, ref_data, var, season)
mv1_p = cdutil.averager(mv1_p, axis="x")
mv2_p = cdutil.averager(mv2_p, axis="x")
parameter.output_file = "-".join(
[ref_name, var, season, parameter.regions[0]])
parameter.main_title = str(" ".join([var, season]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
if len(mv1_p.getLatitude()) < len(mv2_p.getLatitude()):
mv1_reg = mv1_p
lev_out = mv1_p.getLevel()
lat_out = mv1_p.getLatitude()
mv2_reg = mv2_p.crossSectionRegrid(lev_out, lat_out)
# Apply the mask back, since crossSectionRegrid
# doesn't preserve the mask.
mv2_reg = MV2.masked_where(mv2_reg == mv2_reg.fill_value,
mv2_reg)
elif len(mv1_p.getLatitude()) > len(mv2_p.getLatitude()):
mv2_reg = mv2_p
lev_out = mv2_p.getLevel()
lat_out = mv2_p.getLatitude()
mv1_reg = mv1_p.crossSectionRegrid(lev_out, lat_out)
# Apply the mask back, since crossSectionRegrid
# doesn't preserve the mask.
mv1_reg = MV2.masked_where(mv1_reg == mv1_reg.fill_value,
mv1_reg)
else:
mv1_reg = mv1_p
mv2_reg = mv2_p
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_p, mv1_p, mv2_reg, mv1_reg,
diff)
parameter.var_region = "global"
plot(
parameter.current_set,
mv2_p,
mv1_p,
diff,
metrics_dict,
parameter,
)
utils.general.save_ncfiles(parameter.current_set, mv1_p, mv2_p,
diff, parameter)
# For variables without a z-axis.
elif mv1.getLevel() is None and mv2.getLevel() is None:
for region in regions:
# print("Selected region: {}".format(region))
mv1_domain = utils.general.select_region(
region, mv1, land_frac, ocean_frac, parameter)
mv2_domain = utils.general.select_region(
region, mv2, land_frac, ocean_frac, parameter)
parameter.output_file = "-".join(
[ref_name, var, season, region])
parameter.main_title = str(" ".join([var, season, region]))
# Regrid towards the lower resolution of the two
# variables for calculating the difference.
mv1_reg, mv2_reg = utils.general.regrid_to_lower_res(
mv1_domain,
mv2_domain,
parameter.regrid_tool,
parameter.regrid_method,
)
# Special case.
if var == "TREFHT_LAND" or var == "SST":
if ref_name == "WILLMOTT":
mv2_reg = MV2.masked_where(
mv2_reg == mv2_reg.fill_value, mv2_reg)
land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
mv1_reg = MV2.masked_where(land_mask, mv1_reg)
mv2_reg = MV2.masked_where(land_mask, mv2_reg)
diff = mv1_reg - mv2_reg
metrics_dict = create_metrics(mv2_domain, mv1_domain,
mv2_reg, mv1_reg, diff)
parameter.var_region = region
plot(
parameter.current_set,
mv2_domain,
mv1_domain,
diff,
metrics_dict,
parameter,
)
utils.general.save_ncfiles(
parameter.current_set,
mv1_domain,
mv2_domain,
diff,
parameter,
)
else:
raise RuntimeError(
"Dimensions of the two variables are different. Aborting.")
return parameter
def run_diag(parameter: "CoreParameter"):
"""Runs the annual cycle zonal mean diagnostic.
:param parameter: Parameters for the run
:type parameter: CoreParameter
:return: Parameters for the run
:rtype: CoreParameter
"""
variables: List[str] = parameter.variables
ref_name = getattr(parameter, "ref_name", "")
test_data = Dataset(parameter, test=True)
ref_data = Dataset(parameter, ref=True)
parameter.test_name_yrs = get_name_and_yrs(parameter, test_data, "01")
parameter.ref_name_yrs = get_name_and_yrs(parameter, ref_data, "01")
for var in variables:
test_ac = _create_annual_cycle(test_data, var)
ref_ac = _create_annual_cycle(ref_data, var)
test_ac_reg, ref_ac_reg = regrid_to_lower_res(
test_ac,
ref_ac,
parameter.regrid_tool,
parameter.regrid_method,
)
test_ac_zonal_mean = cdutil.averager(test_ac,
axis="x",
weights="generate")
test_ac_reg_zonal_mean = cdutil.averager(test_ac_reg,
axis="x",
weights="generate")
if (parameter.ref_name == "OMI-MLS"
): # SCO from OMI-MLS only available as (time, lat)
test_ac_reg_zonal_mean = select_region_lat_lon(
"60S60N", test_ac_reg_zonal_mean, parameter)
test_ac_zonal_mean = select_region_lat_lon("60S60N",
test_ac_zonal_mean,
parameter)
if var == "SCO":
ref_ac_zonal_mean = ref_ac
ref_ac_reg_zonal_mean = ref_ac_reg
else:
ref_ac_zonal_mean = cdutil.averager(ref_ac,
axis="x",
weights="generate")
ref_ac_reg_zonal_mean = cdutil.averager(ref_ac_reg,
axis="x",
weights="generate")
else:
ref_ac_zonal_mean = cdutil.averager(ref_ac,
axis="x",
weights="generate")
ref_ac_reg_zonal_mean = cdutil.averager(ref_ac_reg,
axis="x",
weights="generate")
# if var == 'SCO' and parameter.ref_name=='OMI-MLS': # SCO from OMI-MLS only available as (time, lat)
# ref_ac_zonal_mean = ref_ac
# ref_ac_reg_zonal_mean = ref_ac_reg
# test_ac_reg_zonal_mean = select_region_lat_lon("60S60N", test_ac_reg_zonal_mean, parameter)
# test_ac_zonal_mean = select_region_lat_lon("60S60N", test_ac_zonal_mean, parameter)
# else:
# ref_ac_zonal_mean = cdutil.averager(ref_ac, axis="x", weights="generate")
# ref_ac_reg_zonal_mean = cdutil.averager(
# ref_ac_reg, axis="x", weights="generate"
# )
diff_ac = test_ac_reg_zonal_mean - ref_ac_reg_zonal_mean
diff_ac.setAxis(1, test_ac_reg_zonal_mean.getAxis(1))
diff_ac.setAxis(0, test_ac_reg_zonal_mean.getAxis(0))
parameter.var_id = var
parameter.output_file = "-".join([ref_name, var, "Annual-Cycle"])
parameter.main_title = str(" ".join([var, "Zonel Mean Annual Cycle"]))
parameter.viewer_descr[var] = (test_ac.long_name if hasattr(
test_ac, "long_name") else "No long_name attr in test data.")
metrics_dict: Dict[str, Any] = {}
plot(
parameter.current_set,
ref_ac_zonal_mean,
test_ac_zonal_mean,
diff_ac,
metrics_dict,
parameter,
)
save_ncfiles(
parameter.current_set,
ref_ac_zonal_mean,
test_ac_zonal_mean,
diff_ac,
parameter,
)
return parameter
def run_diag(parameter: "CoreParameter"):
"""Runs the annual cycle zonal mean diagnostic.
:param parameter: Parameters for the run
:type parameter: CoreParameter
:return: Parameters for the run
:rtype: CoreParameter
"""
variables: List[str] = parameter.variables
ref_name = getattr(parameter, "ref_name", "")
test_data = Dataset(parameter, test=True)
ref_data = Dataset(parameter, ref=True)
parameter.test_name_yrs = get_name_and_yrs(parameter, test_data, "01")
parameter.ref_name_yrs = get_name_and_yrs(parameter, ref_data, "01")
for var in variables:
test_ac = _create_annual_cycle(test_data, var)
ref_ac = _create_annual_cycle(ref_data, var)
test_ac_reg, ref_ac_reg = regrid_to_lower_res(
test_ac,
ref_ac,
parameter.regrid_tool,
parameter.regrid_method,
)
test_ac_zonal_mean = cdutil.averager(test_ac,
axis="x",
weights="generate")
ref_ac_zonal_mean = cdutil.averager(ref_ac,
axis="x",
weights="generate")
test_ac_reg_zonal_mean = cdutil.averager(test_ac_reg,
axis="x",
weights="generate")
ref_ac_reg_zonal_mean = cdutil.averager(ref_ac_reg,
axis="x",
weights="generate")
diff_ac = test_ac_reg_zonal_mean - ref_ac_reg_zonal_mean
parameter.var_id = var
parameter.output_file = "-".join([ref_name, var, "Annual-Cycle"])
parameter.main_title = str(" ".join([var, "Zonel Mean Annual Cycle"]))
parameter.viewer_descr[var] = (test_ac.long_name if hasattr(
test_ac, "long_name") else "No long_name attr in test data.")
metrics_dict: Dict[str, Any] = {}
plot(
parameter.current_set,
ref_ac_zonal_mean,
test_ac_zonal_mean,
diff_ac,
metrics_dict,
parameter,
)
save_ncfiles(
parameter.current_set,
ref_ac_zonal_mean,
test_ac_zonal_mean,
diff_ac,
parameter,
)
return parameter
def run_diag(parameter):
reference_data_path = parameter.reference_data_path
test_data_path = parameter.test_data_path
variables = parameter.variables
seasons = parameter.seasons
ref_name = parameter.ref_name
regions = parameter.regions
for season in seasons:
try:
filename1 = utils.get_test_filename(parameter, season)
filename2 = utils.get_ref_filename(parameter, season)
except IOError as e:
print e
# the file for the current parameters wasn't found, move to next parameters
continue
print('test file: {}'.format(filename1))
print('reference file: {}'.format(filename2))
f_mod = cdms2.open(filename1)
f_obs = cdms2.open(filename2)
# save land/ocean fraction for masking
land_frac = f_mod('LANDFRAC')
ocean_frac = f_mod('OCNFRAC')
for var in variables:
print '***********', variables
print var
parameter.var_id = var
mv1 = acme.process_derived_var(var, acme.derived_variables, f_mod,
parameter)
mv2 = acme.process_derived_var(var, acme.derived_variables, f_obs,
parameter)
# special case, cdms didn't properly convert mask with fill value
# -999.0, filed issue with denise
if ref_name == 'WARREN':
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -0.9, mv2)
# following should move to derived variable
if ref_name == 'AIRS':
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 > 1e+20, mv2)
if ref_name == 'WILLMOTT' or ref_name == 'CLOUDSAT':
print mv2.fill_value
# mv2=MV2.masked_where(mv2==mv2.fill_value,mv2)
# this is cdms2 for bad mask, denise's fix should fix
mv2 = MV2.masked_where(mv2 == -999., mv2)
print mv2.fill_value
# following should move to derived variable
if var == 'PRECT_LAND':
days_season = {
'ANN': 365,
'DJF': 90,
'MAM': 92,
'JJA': 92,
'SON': 91
}
# mv1 = mv1 * days_season[season] * 0.1 #following AMWG
# approximate way to convert to seasonal cumulative
# precipitation, need to have solution in derived variable,
# unit convert from mm/day to cm
mv2 = mv2 / days_season[season] / \
0.1 # convert cm to mm/day instead
mv2.units = 'mm/day'
if mv1.getLevel() and mv2.getLevel(): # for variables with z axis:
plev = parameter.plevs
print 'selected pressure level', plev
f_ins = [f_mod, f_obs]
for f_ind, mv in enumerate([mv1, mv2]):
mv_plv = mv.getLevel()
# var(time,lev,lon,lat) convert from hybrid level to pressure
if mv_plv.long_name.lower().find('hybrid') != -1:
f_in = f_ins[f_ind]
hyam = f_in('hyam')
hybm = f_in('hybm')
ps = f_in('PS') #Pa
mv_p = utils.hybrid_to_plevs(mv, hyam, hybm, ps, plev)
elif mv_plv.long_name.lower().find(
'pressure') != -1 or mv_plv.long_name.lower().find(
'isobaric') != -1: # levels are presure levels
mv_p = utils.pressure_to_plevs(mv, plev)
else:
raise RuntimeError(
"Vertical level is neither hybrid nor pressure. Abort"
)
if f_ind == 0:
mv1_p = mv_p
if f_ind == 1:
mv2_p = mv_p
#select plev
for ilev in range(len(plev)):
mv1 = mv1_p[ilev, ]
mv2 = mv2_p[ilev, ]
#select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print "selected region", region
mv1_zonal = cdutil.averager(mv1, axis='x')
mv2_zonal = cdutil.averager(mv2, axis='x')
# Regrid towards lower resolution of two variables for
# calculating difference
print mv1_zonal.shape, mv2_zonal.shape
mv1_reg, mv2_reg = regrid_to_lower_res_1d(
mv1_zonal, mv2_zonal)
diff = mv1_reg - mv2_reg
parameter.output_file = '-'.join([
ref_name, var,
str(int(plev[ilev])), season, region
])
parameter.main_title = str(' '.join(
[var,
str(int(plev[ilev])), 'mb', season, region]))
parameter.var_region = region
plot('3', mv2_zonal, mv1_zonal, diff, {}, parameter)
utils.save_ncfiles('3', mv1_zonal, mv2_zonal, diff,
parameter)
# mv1_domain, mv2_domain = utils.select_region(region, mv1, mv2, land_frac,ocean_frac,parameter)
#
# parameter.output_file = '-'.join(
# [ref_name, var, str(int(plev[ilev])), season, region])
# parameter.main_title = str(
# ' '.join([var, str(int(plev[ilev])), 'mb', season, region]))
#
# # Regrid towards lower resolution of two variables for
# # calculating difference
# mv1_reg, mv2_reg = utils.regrid_to_lower_res(
# mv1_domain, mv2_domain, parameter.regrid_tool, parameter.regrid_method)
#
# # Plotting
# diff = mv1_reg - mv2_reg
# metrics_dict = create_metrics(
# mv2_domain, mv1_domain, mv2_reg, mv1_reg, diff)
#
# parameter.var_region = region
# plot('3', mv2_zonal, mv1_zonal, diff parameter)
# utils.save_ncfiles('5', mv1_domain, mv2_domain, diff, parameter)
# for variables without z axis:
elif mv1.getLevel() == None and mv2.getLevel() == None:
#select region
if len(regions) == 0:
regions = ['global']
for region in regions:
print "selected region", region
mv1_zonal = cdutil.averager(mv1, axis='x')
mv2_zonal = cdutil.averager(mv2, axis='x')
print mv1_zonal.shape, mv2_zonal.shape
mv1_reg, mv2_reg = regrid_to_lower_res_1d(
mv1_zonal, mv2_zonal)
diff = mv1_reg - mv2_reg
parameter.output_file = '-'.join(
[ref_name, var, season, region])
parameter.main_title = str(' '.join([var, season, region]))
parameter.var_region = region
plot('3', mv2_zonal, mv1_zonal, diff, {}, parameter)
utils.save_ncfiles('3', mv1_zonal, mv2_zonal, diff,
parameter)
# mv1_domain, mv2_domain = utils.select_region(region, mv1, mv2, land_frac,ocean_frac,parameter)
#
# parameter.output_file = '-'.join(
# [ref_name, var, season, region])
# parameter.main_title = str(' '.join([var, season, region]))
#
# # regrid towards lower resolution of two variables for
# # calculating difference
# mv1_reg, mv2_reg = utils.regrid_to_lower_res(
# mv1_domain, mv2_domain, parameter.regrid_tool, parameter.regrid_method)
#
# # if var is 'SST' or var is 'TREFHT_LAND': #special case
#
# if var == 'TREFHT_LAND'or var == 'SST': # use "==" instead of "is"
# if ref_name == 'WILLMOTT':
# mv2_reg = MV2.masked_where(
# mv2_reg == mv2_reg.fill_value, mv2_reg)
# print ref_name
#
# # if mv.mask is False:
# # mv = MV2.masked_less_equal(mv, mv._FillValue)
# # print "*************",mv.count()
# land_mask = MV2.logical_or(mv1_reg.mask, mv2_reg.mask)
# mv1_reg = MV2.masked_where(land_mask, mv1_reg)
# mv2_reg = MV2.masked_where(land_mask, mv2_reg)
#
# diff = mv1_reg - mv2_reg
# metrics_dict = create_metrics(
# mv2_domain, mv1_domain, mv2_reg, mv1_reg, diff)
# parameter.var_region = region
# plot('5', mv2_domain, mv1_domain, diff, metrics_dict, parameter)
# utils.save_ncfiles('5', mv1_domain, mv2_domain, diff, parameter)
else:
raise RuntimeError(
"Dimensions of two variables are difference. Abort")
f_obs.close()
f_mod.close()
