def compute(param):
template = populateStringConstructor(args.filename_template, args)
template.variable = param.varname
template.month = param.monthname
fnameRoot = param.fileName
reverted = template.reverse(os.path.basename(fnameRoot))
model = reverted["model"]
print('Specifying latitude / longitude domain of interest ...')
datanameID = 'diurnalstd' # Short ID name of output data
latrange = (param.args.lat1, param.args.lat2)
lonrange = (param.args.lon1, param.args.lon2)
region = cdutil.region.domain(latitude=latrange, longitude=lonrange)
if param.args.region_name == "":
region_name = "{:g}_{:g}&{:g}_{:g}".format(*(latrange + lonrange))
else:
region_name = param.args.region_name
print('Reading %s ...' % fnameRoot)
reverted = template.reverse(os.path.basename(fnameRoot))
model = reverted["model"]
try:
f = cdms2.open(fnameRoot)
x = f(datanameID, region)
units = x.units
print(' Shape =', x.shape)
print('Finding RMS area-average ...')
x = x * x
x = cdutil.averager(x, weights='unweighted')
x = cdutil.averager(x, axis='xy')
x = numpy.ma.sqrt(x)
print('For %8s in %s, average variance of hourly values = (%5.2f %s)^2' % (model, monthname, x, units))
f.close()
except Exception as err:
print("Failed model %s with error: %s" % (model, err))
x = 1.e20
return model, region, {region_name: x}
def calcBias(a, b):
# Calculate bias
# a, b: cdms 2d variables (lat, lon)
result = cdutil.averager(a, axis="xy",
weights="weighted") - cdutil.averager(
b, axis="xy", weights="weighted")
return float(result)
def compute(param):
template = populateStringConstructor(args.filename_template, args)
template.variable = param.varname
template.month = param.monthname
fnameRoot = param.fileName
reverted = template.reverse(os.path.basename(fnameRoot))
model = reverted["model"]
print('Specifying latitude / longitude domain of interest ...')
datanameID = 'diurnalstd' # Short ID name of output data
latrange = (param.args.lat1, param.args.lat2)
lonrange = (param.args.lon1, param.args.lon2)
region = cdutil.region.domain(latitude=latrange, longitude=lonrange)
if param.args.region_name == "":
region_name = "{:g}_{:g}&{:g}_{:g}".format(*(latrange + lonrange))
else:
region_name = param.args.region_name
print('Reading %s ...' % fnameRoot)
reverted = template.reverse(os.path.basename(fnameRoot))
model = reverted["model"]
try:
f = cdms2.open(fnameRoot)
x = f(datanameID, region)
units = x.units
print(' Shape =', x.shape)
print('Finding RMS area-average ...')
x = x * x
x = cdutil.averager(x, weights='unweighted')
x = cdutil.averager(x, axis='xy')
x = numpy.ma.sqrt(x)
print('For %8s in %s, average variance of hourly values = (%5.2f %s)^2' % (model, monthname, x, units))
f.close()
except Exception as err:
print("Failed model %s with error: %s" % (model, err))
x = 1.e20
return model, region, {region_name: x}
def process_u_for_time_height(data_region):
# Average over longitude (i.e., each latitude's average in data_region)
data_lon_average = cdutil.averager(data_region, axis="x")
# Average over latitude (i.e., average for entire data_region)
data_lon_lat_average = cdutil.averager(data_lon_average, axis="y")
# Get data by vertical level
level_data = data_lon_lat_average.getAxis(1)
return data_lon_lat_average, level_data
def createDeterministicFcstMJOinFiles(curday):
# os.chdir(anaanopath)
merged_ana_10daysfcst_anofile = 'um_ana_tseries_and_determisitic_10days_fcst_%s_ano.xml' % curday
merged_ana_10daysfcst_anofpath = os.path.join(
fcstanopath, merged_ana_10daysfcst_anofile)
inf = cdms2.open(merged_ana_10daysfcst_anofpath)
cday = tobj.timestr2comp(curday)
outfile_dfcst = 'um_determisitic_10days_fcst_mjo_project_input_%s.nc' % curday
outfile_dpath = os.path.join(mjoDFcstInpath, outfile_dfcst)
outf_df = cdms2.open(outfile_dpath, 'w')
outfile_ana = 'um_analysis_mjo_project_input_%s.nc' % curday
outfile_apath = os.path.join(mjoAnaInpath, outfile_ana)
outf_a = cdms2.open(outfile_apath, 'w')
for (ivar, lev, longitudunal_norm_factor, ovar) in [
('ulwrf', None, 15.1, 'olr'),
('u', 20000, 4.81, 'u200'),
('u', 85000, 1.81, 'u850'),
]:
for i in range(
0, 11, 1
): # 0 will bring past 120 days analysis and 1 to 10 will bring 10 days forecast
nextday = tobj.moveTime(cday.year, cday.month, cday.day, i)
previous120day = tobj.moveTime(
nextday.year, nextday.month, nextday.day,
-119) # here -119 will bring previous120day
previous120data = inf(ivar,
time=(previous120day, nextday),
level=lev)
print "previous120data", i, previous120day, nextday, previous120data.shape
previous120data_mean = cdutil.averager(previous120data, axis='t')
if i: # forecast individual 10 days in loop
nextdaydata = inf(ivar, time=nextday, level=lev)
axlist = nextdaydata.getAxisList()
nextdaydata -= previous120data_mean # subtract previous 120 days (119 days analysis + 1st day fcst)
else: # past 120 days analysis
axlist = previous120data.getAxisList()
nextdaydata = previous120data - previous120data_mean # just subtract 120 days mean of analysis
# end of if i:
#
subtropical_data = nextdaydata(latitude=(-15, 15, 'ccb'))
subtropical_data_avg = cdutil.averager(subtropical_data, axis='y')
subtropical_data_avg /= longitudunal_norm_factor
fdata = cdms2.createVariable(subtropical_data_avg, id=ovar)
axlist = axlist[0:-2] + [
axlist[-1]
] # i.e omitting latitude axis, since we already averaged over that axis.
fdata.setAxisList(axlist)
if i:
outf_df.write(fdata) # forecast file
else:
outf_a.write(fdata) # analysis file
# end of for i in range(1, 11, 1):
# end of for (ivar, ....):
outf_df.close()
outf_a.close()
inf.close()
def applyOperation( self, input_variable, operation ):
result = None
try:
self.setTimeBounds( input_variable )
operator = None
# pydevd.settrace('localhost', port=8030, stdoutToServer=False, stderrToServer=True)
wpsLog.debug( " $$$ ApplyOperation: %s " % str( operation ) )
if operation is not None:
type = operation.get('type','').lower()
bounds = operation.get('bounds','').lower()
op_start_time = time.clock() # time.time()
if not bounds:
if type == 'departures':
ave = cdutil.averager( input_variable, axis='t', weights='equal' )
result = input_variable - ave
elif type == 'climatology':
result = cdutil.averager( input_variable, axis='t', weights='equal' )
else:
result = input_variable
time_axis = input_variable.getTime()
elif bounds == 'np':
if type == 'departures':
result = ma.anomalies( input_variable ).squeeze()
elif type == 'climatology':
result = ma.average( input_variable ).squeeze()
else:
result = input_variable
time_axis = input_variable.getTime()
else:
if bounds == 'djf': operator = cdutil.DJF
elif bounds == 'mam': operator = cdutil.MAM
elif bounds == 'jja': operator = cdutil.JJA
elif bounds == 'son': operator = cdutil.SON
elif bounds == 'year': operator = cdutil.YEAR
elif bounds == 'annualcycle': operator = cdutil.ANNUALCYCLE
elif bounds == 'seasonalcycle': operator = cdutil.SEASONALCYCLE
if operator <> None:
if type == 'departures': result = operator.departures( input_variable ).squeeze()
elif type == 'climatology': result = operator.climatology( input_variable ).squeeze()
else: result = operator( input_variable ).squeeze()
time_axis = result.getTime()
op_end_time = time.clock() # time.time()
wpsLog.debug( " ---> Base Operation Time: %.5f" % (op_end_time-op_start_time) )
else:
result = input_variable
time_axis = input_variable.getTime()
if isinstance( result, float ):
result_data = [ result ]
elif result is not None:
if result.__class__.__name__ == 'TransientVariable':
result = ma.masked_equal( result.squeeze().getValue(), input_variable.getMissing() )
result_data = result.tolist( numpy.nan )
else: result_data = None
except Exception, err:
wpsLog.debug( "Exception applying Operation '%s':\n %s" % ( str(operation), traceback.format_exc() ) )
return ( None, None )
def run_diag_scatter(parameter):
variables = parameter.variables
run_type = parameter.run_type
# We will always use the same regions, so we don't do the following:
# x['region'] = parameter.nino_region
# y['region'] = parameter.regions[0]
x = {"var": "TS", "units": "degC", "region": "NINO3"}
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
if parameter.print_statements:
print("run_type: {}".format(run_type))
if run_type == "model_vs_model":
x["test"] = calculate_nino_index_model(test_data, x["region"],
parameter)
x["ref"] = calculate_nino_index_model(ref_data, x["region"], parameter)
elif run_type == "model_vs_obs":
x["test"] = calculate_nino_index_model(test_data, x["region"],
parameter)
x["ref"] = calculate_nino_index(x["region"], parameter, ref=True)
else:
raise Exception("Invalid run_type={}".format(run_type))
parameter.test_name_yrs = utils.general.get_name_and_yrs(
parameter, test_data)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(
parameter, ref_data)
for y_var in variables:
if y_var == "TAUX":
regions = ["NINO4"]
else:
regions = ["NINO3"]
for region in regions:
y = {"var": y_var, "region": region}
test_data_ts = test_data.get_timeseries_variable(y_var)
ref_data_ts = ref_data.get_timeseries_variable(y_var)
y_region = default_regions.regions_specs[region][
"domain"] # type: ignore
test_data_ts_regional = test_data_ts(y_region)
ref_data_ts_regional = ref_data_ts(y_region)
# Domain average
test_avg = cdutil.averager(test_data_ts_regional, axis="xy")
ref_avg = cdutil.averager(ref_data_ts_regional, axis="xy")
# Get anomaly from annual cycle climatology
y["test"] = cdutil.ANNUALCYCLE.departures(test_avg)
y["ref"] = cdutil.ANNUALCYCLE.departures(ref_avg)
y["units"] = test_avg.units
if y_var == "TAUX":
y["test"] *= 1000
y["ref"] *= 1000
y["units"] = "10^3 {}".format(y["units"])
parameter.var_id = "{}-feedback".format(y["var"])
title_tuple = (y["var"], y["region"], x["var"], x["region"])
parameter.main_title = "{} anomaly ({}) vs. {} anomaly ({})".format(
*title_tuple)
parameter.viewer_descr[y["var"]] = parameter.main_title
parameter.output_file = "feedback-{}-{}-{}-{}".format(*title_tuple)
plot_scatter(x, y, parameter)
return parameter
def get_residual_timeseries(timeseries_ano,
mode,
region_subdomain,
RmDomainMean=True):
"""
NOTE: Calculate residual by subtracting domain average (or global mean)
Input
- timeseries_ano: anomaly time series, cdms2 array, 3d (t, y, x)
- mode: string, mode name, must be defined in regions_specs
- RmDomainMean: bool (True or False).
If True, remove domain mean of each time step.
Ref:
Bonfils and Santer (2011)
https://doi.org/10.1007/s00382-010-0920-1
Bonfils et al. (2015)
https://doi.org/10.1175/JCLI-D-15-0341.1
If False, remove global mean of each time step for PDO, or
do nothing for other modes
Default is True for this function.
- region_subdomain: lat lon range of sub domain for given mode, which was
extracted from regions_specs -- that is a dict contains domain
lat lon ragne for given mode
Output
- timeseries_residual: cdms2 array, 3d (t, y, x)
"""
if RmDomainMean:
# Get domain mean
regional_ano_mean_timeseries = cdutil.averager(
timeseries_ano(region_subdomain), axis='xy', weights='weighted')
# Match dimension
timeseries_ano, regional_ano_mean_timeseries = \
genutil.grower(
timeseries_ano, regional_ano_mean_timeseries)
# Subtract domain mean
timeseries_residual = MV2.subtract(timeseries_ano,
regional_ano_mean_timeseries)
else:
if mode in ['PDO', 'NPGO']:
# Get global mean
global_ano_mean_timeseries = cdutil.averager(
timeseries_ano(latitude=(-60, 70)),
axis='xy',
weights='weighted')
# Match dimension
timeseries_ano, global_ano_mean_timeseries = \
genutil.grower(
timeseries_ano, global_ano_mean_timeseries)
# Subtract global mean
timeseries_residual = MV2.subtract(timeseries_ano,
global_ano_mean_timeseries)
else:
timeseries_residual = timeseries_ano
# return result
return timeseries_residual
def compute(dm, do):
""" Computes ZONAL MEAN assumes rectilinear/regular grid"""
if dm is None and do is None: # just want the doc
return {
"Name": "Zonal Mean",
"Abstract": "Compute Zonal Mean",
"URI": "http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "*****@*****.**",
"Comments": ""
}
return cdutil.averager(dm, axis='x'), cdutil.averager(do, axis='x')
def compute(dm, do):
""" Computes ZONAL MEAN assumes rectilinear/regular grid"""
if dm is None and do is None: # just want the doc
return {
"Name": "Zonal Mean",
"Abstract": "Compute Zonal Mean",
"URI":
"http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "*****@*****.**",
"Comments": ""
}
return cdutil.averager(dm, axis='x'), cdutil.averager(do, axis='x')
def GetData(idx):
if idx == 1: fopen=cdms.open('tas_ECMWFear20c_1901_2010.nc'); DATAsp = fopen('ECMWFear20c_EquArea')[79:]
if idx == 2: fopen=cdms.open('tas_NOAA20CRv3_1901_2010.nc'); DATAsp = fopen('NOAA20CRv3_EquArea') [79:]
if idx == 3: fopen=cdms.open('tas_NCEPpNCAR_1948_2010.nc'); DATAsp = fopen('NCEPpNCAR_EquArea')[32:]
if idx == 4: fopen=cdms.open('tas_JRA55_1958_2013.nc'); DATAsp = fopen('JRA55_EquArea')[22:-3]
if idx == 5: fopen=cdms.open('tp_CFSR_detrended.nc'); DATAsp = fopen('tas')[:-4]
if idx == 6: fopen=cdms.open('tp_ERA5_detrended.nc'); DATAsp = fopen('tas')[:-4]
if idx == 7: fopen=cdms.open('tp_MERRA2_detrended.nc'); DATAsp = fopen('tas')[:-4]
fopen.close()
DATAgm = cdutil.averager(DATAsp,axis='yx')
DATAgmCent = np.array(DATAgm - cdutil.averager(DATAgm, axis=0, weight='equal'))
return DATAsp, np.array(DATAgm), np.array(DATAgmCent)
def run_diag_scatter(parameter):
variables = parameter.variables
run_type = parameter.run_type
# We will always use the same regions, so we don't do the following:
# x['region'] = parameter.nino_region
# y['region'] = parameter.regions[0]
x = {'var': 'TS', 'units': 'degC', 'region': 'NINO3'}
test_data = utils.dataset.Dataset(parameter, test=True)
ref_data = utils.dataset.Dataset(parameter, ref=True)
if parameter.print_statements:
print('run_type: {}'.format(run_type))
if run_type == 'model_vs_model':
x['test'] = calculate_nino_index_model(test_data, x['region'], parameter)
x['ref'] = calculate_nino_index_model(ref_data, x['region'], parameter)
elif run_type == 'model_vs_obs':
x['test'] = calculate_nino_index_model(test_data, x['region'], parameter)
x['ref'] = calculate_nino_index(x['region'], parameter, ref=True)
else:
raise Exception('Invalid run_type={}'.format(run_type))
parameter.test_name_yrs = utils.general.get_name_and_yrs(parameter, test_data)
parameter.ref_name_yrs = utils.general.get_name_and_yrs(parameter, ref_data)
for y_var in variables:
if y_var == 'TAUX':
regions = ['NINO4']
else:
regions = ['NINO3']
for region in regions:
y = {'var': y_var, 'region': region}
test_data_ts = test_data.get_timeseries_variable(y_var)
ref_data_ts = ref_data.get_timeseries_variable(y_var)
y_region = default_regions.regions_specs[region]['domain']
test_data_ts_regional = test_data_ts(y_region)
ref_data_ts_regional = ref_data_ts(y_region)
# Domain average
test_avg = cdutil.averager(test_data_ts_regional, axis='xy')
ref_avg = cdutil.averager(ref_data_ts_regional, axis='xy')
# Get anomaly from annual cycle climatology
y['test'] = cdutil.ANNUALCYCLE.departures(test_avg)
y['ref'] = cdutil.ANNUALCYCLE.departures(ref_avg)
y['units'] = test_avg.units
if y_var == 'TAUX':
y['test'] *= 1000
y['ref'] *= 1000
y['units'] = '10^3 {}'.format(y['units'])
parameter.var_id = '{}-feedback'.format(y['var'])
title_tuple = (y['var'], y['region'], x['var'], x['region'])
parameter.main_title = '{} anomaly ({}) vs. {} anomaly ({})'.format(*title_tuple)
parameter.viewer_descr[y['var']] = parameter.main_title
parameter.output_file = 'feedback-{}-{}-{}-{}'.format(*title_tuple)
plot_scatter(x, y, parameter)
return parameter
def compute(dm, do):
""" Computes ANNUAL MEAN"""
if dm is None and do is None: # just want the doc
return {
"Name": "Annual Mean",
"Abstract": "Compute Annual Mean",
"URI": "http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "*****@*****.**",
"Comments": "Assumes input are 12 months climatology"
}
# Do we really want this? Wouldn't it better to let it fails
cdms2.setAutoBounds('on')
return cdutil.averager(dm, axis='t'), cdutil.averager(do, axis='t')
def compute(dm, do):
""" Computes ANNUAL MEAN"""
if dm is None and do is None: # just want the doc
return {
"Name": "Annual Mean",
"Abstract": "Compute Annual Mean",
"URI":
"http://uvcdat.llnl.gov/documentation/" +
"utilities/utilities-2.html",
"Contact": "Peter Gleckler <*****@*****.**>",
"Comments": "Assumes input are 12 months climatology"
}
# Do we really want this? Wouldn't it better to let it fails
cdms2.setAutoBounds('on')
return cdutil.averager(dm, axis='t'), cdutil.averager(do, axis='t')
def arbitrary_checking(mode, eof_Nth):
"""
NOTE: To keep sign of EOF pattern consistent across observations or models,
this function check whether multiplying -1 to EOF pattern and PC
is needed or not
Input
- mode: string, modes of variability. e.g., 'PDO', 'PNA', 'NAM', 'SAM'
- eof_Nth: cdms2 array from eofs, eof pattern
Ouput
- reverse_sign: bool, True or False
"""
reverse_sign = False
# Explicitly check average of geographical region for each mode
if mode == 'PDO':
if float(
cdutil.averager(eof_Nth(latitude=(30, 40),
longitude=(150, 180)),
axis='xy',
weights='weighted')) >= 0:
reverse_sign = True
elif mode == 'PNA':
if float(
cdutil.averager(eof_Nth(latitude=(80, 90)),
axis='xy',
weights='weighted')) <= 0:
reverse_sign = True
elif mode == 'NAM' or mode == 'NAO':
if float(
cdutil.averager(eof_Nth(latitude=(60, 80)),
axis='xy',
weights='weighted')) >= 0:
reverse_sign = True
elif mode == 'SAM':
if float(
cdutil.averager(eof_Nth(latitude=(-60, -90)),
axis='xy',
weights='weighted')) >= 0:
reverse_sign = True
else: # Minimum sign control part was left behind for any future usage..
if float(eof_Nth[-1][-1]) is not eof_Nth.missing:
if float(eof_Nth[-1][-1]) >= 0:
reverse_sign = True
elif float(eof_Nth[-2][-2]) is not eof_Nth.missing:
if float(eof_Nth[-2]
[-2]) >= 0: # Double check missing value at pole
reverse_sign = True
# return result
return reverse_sign
def average_vs_projection(data):
"""Show the difference between PDSI regional average and projections"""
#start = cdtime.comptime(1975,8,1)
#stop = cdtime.comptime(2005,12,31)
start = cdtime.comptime(1900,1,1)
stop = cdtime.comptime(1949,12,31)
ax1=plt.subplot(211)
ax2=plt.subplot(212)
for thing in sorted(["MXDA","OWDA","NADA","MADA","ANZDA"])+["ALL"]:
X=getattr(data,thing)
pdsi_av = cdutil.averager(X.obs,axis='xy')(time=(start,stop))
c=colorregions(X.name)
Plotting.time_plot(pdsi_av,lw=3,color=c,label=X.name,ax=ax1)
Plotting.time_plot(X.projection(time=(start,stop)),lw=3,color=c,label=X.name,ax=ax2)
if thing =="ALL":
x=np.arange(1975,2006)
y=X.projection(time=(start,stop))
p=np.polyfit(x,y,1)
ax2.plot(x,np.polyval(p,x),"k--",lw=1)
y=pdsi_av
p=np.polyfit(x,y,1)
ax1.plot(x,np.polyval(p,x),"k--",lw=1)
ax1.set_title("(a): Regional mean PDSI")
ax1.set_ylabel("PDSI")
ax2.set_title("(b): Projection on fingerprint")
# ax2.set_ylim(-22,22)
ax2.set_ylabel("Projection")
ax1.set_ylim(-1.6,1.6)
plt.legend(ncol=4,fontsize=8)
def TOAswcs(HISSSP_3_110, model_name):
vORG_110 = HISSSP_3_110.vORG
HISm_mean_110 = HISSSP_3_110.HISm_mean
HISm_spat_110 = HISSSP_3_110.HISm_spat
EOFreturn1 = HISSSP_3_110.class_cal_eofs(vORG_110,
HISm_mean_110,
HISm_spat_110,
idx=[-1],
REA_idx=True)
STDeof3 = EOFreturn1['std_EOFsOnHISm'][:, 3]
vars_list = ['rsut', 'rsutcs']
STDsw = []
mondel_list = ['CNRM-CM6-1', 'CNRM-ESM2-1', 'MIROC-ES2L', 'UKESM1-0-LL']
for idx1 in model_name:
if idx1 in mondel_list:
fopen2 = cdms.open(new_data_path + 'Returned_' + vars_list[1] +
'_' + idx1 + '_historical_r1i1p1f2.nc')
else:
fopen2 = cdms.open(new_data_path + 'Returned_' + vars_list[1] +
'_' + idx1 + '_historical_r1i1p1f1.nc')
rsutcs = fopen2('tEquArea', squeeze=1)
used = cdutil.averager(rsutcs[-114:-4, -3:, :], axis='yx')
std = np.std(used)
STDsw.append(std)
STDsw = np.array(STDsw)
fitT = sm.OLS(STDeof3, np.c_[np.ones(17), STDsw]).fit()
pred = fitT.get_prediction(np.c_[np.ones(17), STDsw]).predicted_mean
plt.scatter(STDsw, STDeof3)
plt.plot(STDsw, pred, color='black')
plt.xlim(0.6, 2.2)
plt.ylim(0, 0.25)
plt.show()
plt.clf()
def reduce2latlon_seasonal( mv, seasons=seasonsyr, vid=None ):
"""as reduce2lat_seasonal, but both lat and lon axes are retained.
"""
# This differs from reduce2lat_seasonal only in the line "axis_names ="....
# I need to think about how to structure the code so there's less cut-and-paste!
if vid==None:
vid = 'reduced_'+mv.id
# Note that the averager function returns a variable with meaningless id.
# The climatology function returns the same id as mv, which we also don't want.
# The slicers in time.py require getBounds() to work.
# If it doesn't, we'll have to give it one.
# Setting the _bounds_ attribute will do it.
timeax = timeAxis(mv)
if timeax.getBounds()==None:
timeax._bounds_ = timeax.genGenericBounds()
mvseas = seasons.climatology(mv)
axes = allAxes( mv )
axis_names = [ a.id for a in axes if a.id!='lat' and a.id!='lon' and a.id!='time']
axes_string = '('+')('.join(axis_names)+')'
if len(axes_string)>2:
for axis in mvseas.getAxisList():
if axis.getBounds() is None:
axis._bounds_ = axis.genGenericBounds()
avmv = averager( mvseas, axis=axes_string )
else:
avmv = mvseas
if avmv is None: return avmv
avmv.id = vid
if hasattr(mv,'units'): avmv.units = mv.units
avmv = delete_singleton_axis( avmv, vid='time' )
avmv.units = mv.units
return avmv
def reduce2levlat_seasonal( mv, seasons=seasonsyr, vid=None ):
"""as reduce2levlat, but data is averaged only for time restricted to the specified season;
as in reduce2lat_seasona."""
if vid==None: # Note that the averager function returns a variable with meaningless id.
vid = 'reduced_'+mv.id
if levAxis(mv) is None: return None
if latAxis(mv) is None: return None
axes = allAxes( mv )
timeax = timeAxis(mv)
if timeax.getBounds()==None:
timeax._bounds_ = timeax.genGenericBounds()
if timeax.units=='months':
# Special check necessary for LEGATES obs data, because
# climatology() won't accept this incomplete specification
timeax.units = 'months since 0001-01-01'
mvseas = seasons.climatology(mv)
axis_names = [ a.id for a in axes if a.id!='lev' and a.id!='lat' and a.id!='time']
axes_string = '('+')('.join(axis_names)+')'
if len(axes_string)>2:
avmv = averager( mvseas, axis=axes_string )
else:
avmv = mvseas
avmv.id = vid
avmv = delete_singleton_axis( avmv, vid='time' )
avmv.units = mv.units
return avmv
def testContiguousRegridNANIssue(self):
a = MV2.reshape(MV2.sin(MV2.arange(20000)), (2, 1, 100, 100))
lon = cdms2.createAxis(MV2.arange(100) * 3.6)
lon.designateLongitude()
lon.units = "degrees_east"
lon.id = "longitude"
lat = cdms2.createAxis(MV2.arange(100) * 1.8 - 90.)
lat.id = "latitude"
lat.designateLatitude()
lat.units = "degrees_north"
lev = cdms2.createAxis([1000.])
lev.id = "plev"
lev.designateLevel()
lev.units = "hPa"
t = cdms2.createAxis([0, 31.])
t.id = "time"
t.designateTime()
t.units = "days since 2014"
cdutil.setTimeBoundsMonthly(t)
a.setAxisList((t, lev, lat, lon))
a = MV2.masked_less(a, .5)
grd = cdms2.createGaussianGrid(64)
a = a.ascontiguous()
a = a.regrid(grd, regridTool="regrid2")
a = cdutil.averager(a, axis='txy')
self.assertEqual(a[0], 0.7921019540305255)
def compute(self):
variable = self.getInputFromPort("tvariable")
axis = self.getInputFromPort("axis")
avg = cdutil.averager(variable.data, axis=axis)
self.setResult('tvariable', TransientVariable(avg))
def get_predictor1(data_path, predictor_source, varMEAN, length, selec):
model_flg = 1
if predictor_source == 'models_historical' or predictor_source == 'models_ssp':
for instance in CMIP6_models.instances:
if 'historical' in instance.CaseList and 'ssp585' in instance.CaseList:
if predictor_source == 'models_historical':
fopen = cdms.open(data_path + 'tp_' + instance.Name +
'_historical_' + instance.VarLab[1][0] +
'.nc')
vUsed = fopen(varMEAN)[int(-1 * length - 4):-4]
if predictor_source == 'models_ssp':
fopen = cdms.open(data_path + 'tp_' + instance.Name +
'_ssp585_' + instance.VarLab[1][0] +
'.nc')
vUsed = fopen(varMEAN)[int(-1 * length):]
fopen.close()
vUsed_masked = vUsed - cdutil.averager(
vUsed, axis=0, weights='equal')
if model_flg == 1:
array_MEAN = [vUsed_masked]
lat, lon = vUsed_masked.getAxis(1), vUsed_masked.getAxis(2)
model_flg += 1
else:
array_MEAN = np.ma.concatenate(
[array_MEAN, [vUsed_masked]])
return_ME = MV.array(array_MEAN)
return_ME.setAxis(2, lat)
return_ME.setAxis(3, lon)
return return_ME
def timeseries_average( data_slice ):
# logger = getLogger( "log-map" )
lat, lon = location['latitude'], location['longitude']
timeseries = data_slice(latitude=(lat, lat, "cob"), longitude=(lon, lon, "cob"))
result = cdutil.averager( timeseries, axis='t', weights='equal' ).squeeze().tolist()
# if logger: logger.debug( "Result = %s", str(result) )
return result
def test1dLevelAxis(self):
f = cdms2.open(os.path.join(vcs.sample_data, "ta_ncep_87-6-88-4.nc"))
ta = f("ta", time=slice(0, 1), squeeze=1)
ta = cdutil.averager(ta, axis="yx")
self.x.plot(ta, bg=self.bg)
fnm = os.path.splitext(os.path.split(__file__)[1])[0] + ".png"
self.checkImage(fnm)
def executeOperation(self, task, _input):
self.logger.info("Executing AverageKernel, input metadata = " +
str(_input.metadata))
dset_address = _input.metadata.get("uri",
_input.metadata.get("dataPath"))
vname = _input.metadata.get("name")
jobId = str(task.metadata.get('jobId', task.rId))
dset = cdms2.open(dset_address)
selector = _input.getSelector(dset[vname])
self.logger.info("exec *EXT* AverageKernel, selector: " +
str(selector))
variable = dset(vname, **selector)
axes = task.metadata.get("axis", "xy")
# weights = task.metadata.get( "weights", "" ).split(",")
# if weights == [""]: weights = [ ("generate" if( axis == 'y' ) else "equal") for axis in axes ]
weights = task.metadata.get("weights", "generate").split(",")
if (len(weights) == 1): weights = weights[0]
action = task.metadata.get("action", "average")
returned = 0
result_var = cdutil.averager(variable,
axis=axes,
weights=weights,
action=action,
returned=returned)
self.logger.info("Computed result, input shape = " +
str(variable.shape) + ", output shape = " +
str(result_var.shape))
rv = self.createResult(jobId, result_var, _input, task)
self.logger.info("Result data, shape = " + str(result_var.shape) +
", data = " + np.array_str(rv.array()))
return rv
def gain_pcs_fraction(full_field, eof_pattern, pcs, debug=False):
"""
NOTE: This function is designed for getting fraction of variace obtained by
pseudo pcs
Input: (dimension x, y, t should be identical for above inputs)
- full_field (t,y,x)
- eof_pattern (y,x)
- pcs (t)
Output:
- fraction: cdms2 array but for 1 single number which is float.
Preserve cdms2 array type for netCDF recording.
fraction of explained variance
"""
# 1) Get total variacne ---
variance_total = genutil.statistics.variance(full_field, axis="t")
variance_total_area_ave = cdutil.averager(
variance_total, axis="xy", weights="weighted"
)
# 2) Get variance for pseudo pattern ---
# 2-1) Reconstruct field based on pseudo pattern
if debug:
print("from gain_pcs_fraction:")
print("full_field.shape (before grower): ", full_field.shape)
print("eof_pattern.shape (before grower): ", eof_pattern.shape)
# Extend eof_pattern (add 3rd dimension as time then copy same 2d value for all time step)
reconstructed_field = genutil.grower(full_field, eof_pattern)[
1
] # Matching dimension (add time axis)
for t in range(0, len(pcs)):
reconstructed_field[t] = MV2.multiply(reconstructed_field[t], pcs[t])
# 2-2) Get variance of reconstructed field
variance_partial = genutil.statistics.variance(reconstructed_field, axis="t")
variance_partial_area_ave = cdutil.averager(
variance_partial, axis="xy", weights="weighted"
)
# 3) Calculate fraction ---
fraction = MV2.divide(variance_partial_area_ave, variance_total_area_ave)
# debugging
if debug:
print("full_field.shape (after grower): ", full_field.shape)
print("reconstructed_field.shape: ", reconstructed_field.shape)
print("variance_partial_area_ave: ", variance_partial_area_ave)
print("variance_total_area_ave: ", variance_total_area_ave)
print("fraction: ", fraction)
print("from gain_pcs_fraction done")
# return result
return fraction
def zon_p_plots(var, dsetA, dsetB, title="", nlev=None):
# Average Zonally over last 20 years
bt = 30 * 12
controlV = dsetA(var, time=slice(bt, 600))
compareV = dsetB(var, time=slice(bt, 600))
a = cdutil.averager(controlV, axis="x")
b = cdutil.averager(compareV, axis="x")
c = b - a
cdutil.setTimeBoundsMonthly(c)
plotme = cdutil.DJF.climatology(c)
lat_pressure_contour_cdms(plotme, tit="%s DJF" % title, nlev=nlev)
plotme = cdutil.JJA.climatology(c)
lat_pressure_contour_cdms(plotme, tit="%s JJA" % title, nlev=nlev)
def getProcessedVariable( self, varname, var_proc_op ):
var = self.df[ varname ]
if isNone( var ):
print>>sys.stderr, "Error, can't find variable '%s' in data file." % ( varname )
return None
if var_proc_op == "anomaly_t":
var_ave = cdutil.averager( var, axis='time' )
return var
def weightedAvgAxisPoints(self, var):
""" Update the variable by applying the weighted avg operation to the
the given axis.
"""
varID = var.id
var = cdutil.averager(var, axis="(%s)" % self.axis.id)
var.id = varID
return var
def execute(self):
dataIn=self.loadData()[0]
data,cdms2keyargs = self.loadVariable(dataIn)
dims = "".join(["(%s)" % x for x in cdms2keyargs.keys()])
data = cdutil.averager(data,axis=dims)
data.id=self.getVariableName(dataIn)
self.saveVariable(data,self.average,"json")
return
def EIS_LTS(T,P,RELHUM):
#Calculates the Lower Tropospheric Stability (LTS) and Estimated Inversion Strenght (EIS)
Theta_700=cdu.logLinearInterpolation(T,P,levels=70000)*(1000./700.)**(2./7.)
Theta_700=cdu.averager(Theta_700,axis='z')
Theta_1000=cdu.logLinearInterpolation(T,P,levels=100000)
Theta_1000=cdu.averager(Theta_1000,axis='z')
LTS=Theta_700-Theta_1000
LTS.id='LTS'
LTS.long_name='Lower tropospheric stability'
LTS.units='K'
Theta_700.long_name='Potential temperature at 700hPa'
Theta_700.units='K'
Theta_1000.long_name='Potential temperature at 1000hPa'
Theta_1000.units='K'
# !!! Still need to develop EIS !!!
EIS=LTS
return LTS,EIS,Theta_700,Theta_1000
def decade_fast_annual_cycle(X,debug=False,semiann=False,zonal_average=False,return_Pdays=True):
if len(X.shape)==4:
nt,nlat,nlon,nmod = X.shape
has_models=True
elif len(X.shape)==3:
nt,nlat,nlon = X.shape
has_models = False
if zonal_average:
X = cdutil.averager(X,axis='x')
nyears = nt/60
newshape = (nyears, 60) + X.shape[1:]
print newshape
yrs = X.reshape(newshape)
if semiann:
vec_cycle=broadcast(get_semiannual_cycle)
else:
vec_cycle = broadcast(get_cycle)
# print "vectorized"
apply_everywhere = np.apply_along_axis(vec_cycle,1,yrs)
R = MV.array(apply_everywhere[:,0])
P = MV.array(apply_everywhere[:,1])
#print "got R and P"
if debug:
return R,P
time = X.getTime()[::60]
tax = cdms.createAxis(time)
tax.designateTime()
tax.id = "time"
atts = X.getTime().attributes
for k in atts:
setattr(tax,k,atts[k])
print "got new time"
axlist = [tax]+X.getAxisList()[1:]
R.setAxisList(axlist)
P.setAxisList(axlist)
if return_Pdays is False:
return R,P
#Pnew = P.copy()
#Pold = P.copy()
Pnew = MV.zeros(P.shape)
if has_models:
for mod_i in range(nmod):
chunk = P[:,:,:,mod_i]
Pnew[:,:,:,mod_i] = correct_phase(chunk)
else:
chunk = P[:,:,:]
Pnew = correct_phase(chunk)
Pnew = MV.array(Pnew)
Pnew.setAxisList(axlist)
return R,P,Pnew
def alteredWeightedAvgAxisPoints(self, var):
""" Update the variable by applying the altered weighted avg operation
to the given axis.
"""
varID = var.id
var = cdutil.averager(var, axis="(%s)" % self.axis.id,
weight=self.alteredWeightsVar.filled())
var.id = varID
return var
def getProcessedVariable(self, varname, var_proc_op):
var = self.df[varname]
if isNone(var):
print >> sys.stderr, "Error, can't find variable '%s' in data file." % (
varname)
return None
if var_proc_op == "anomaly_t":
var_ave = cdutil.averager(var, axis='time')
return var
def compute(d):
""" Computes bias"""
if d is None: # just want the doc
return {
"Name": "Mean",
"Abstract": "Area Mean (area weighted)",
"Contact": "*****@*****.**",
}
return MV2.float(cdutil.averager(d, axis='xy', weights='weighted'))
def compute(d):
"""Computes bias"""
if d is None: # just want the doc
return {
"Name": "Mean",
"Abstract": "Area Mean (area weighted)",
"Contact": "*****@*****.**",
}
return MV2.float(cdutil.averager(d, axis="xy", weights="weighted"))
def sumAxisPoints(self, var):
""" Update the variable by applying the sum operation to the given
axis. Note this function is called only when a variable is defined.
"""
varID = var.id
var = cdutil.averager(var, axis="(%s)" % self.axis.id, weight='equal',
action='sum')
var.id = varID
return var
def get_ssp(fname_ssp):
fopen_ssp = cdms.open(data_path + fname_ssp)
v_SSP = fopen_ssp(varMEAN)
v_SSP_Mean = cdutil.averager(v_SSP,
axis='yx') - v_HIS_MeanYr
v_SSP_DeTr = np.convolve(v_SSP_Mean,
np.ones(21) / 21,
mode='valid')
fopen_ssp.close()
HIS_full_Mean = cdutil.averager(v_HIS_full,
axis='yx') - v_HIS_MeanYr
SSP_full_Mean = cdutil.averager(v_SSP,
axis='yx') - v_HIS_MeanYr
COM_full_Mean = np.r_[HIS_full_Mean, SSP_full_Mean]
COM_full_DeTr = np.convolve(COM_full_Mean,
np.ones(21) / 21,
mode='valid')
return COM_full_DeTr, v_SSP_DeTr
def plot(self, prevWindowId, varSpecs, method, template, width, height, opts={}):
canvas = self._canvas[prevWindowId] if prevWindowId != 0 else None
plot = VcsPlot(canvas, width=width, height=height)
plot.setGraphicsMethod(method)
plot.setTemplate(template)
all_vars = []
for varSpec in varSpecs:
if 'json' in varSpec:
f = None
var = compute_graph.loadjson(varSpec['json']).derive()
else:
f = cdms2.open(varSpec['uri'])
# use [] so that the var is not read.
var = f[varSpec['variable']]
if ('operations' in varSpec):
for op in varSpec['operations']:
if ('subRegion' in op):
kargs = op['subRegion']
var = var.subRegion(**kargs)
elif ('subSlice' in op):
kargs = op['subSlice']
# fill in None with begin and end of the current axis
for axis in list(kargs.keys()):
values = kargs[axis]
newValues = values
axisIndex = var.getAxisIndex(axis)
if values[0] is None:
newValues[0] = 0
if values[1] is None:
newValues[1] = var.shape[axisIndex] - 1
kargs[axis] = slice(*newValues)
var = var.subSlice(**kargs)
elif ('transform' in op):
for axis in op["transform"]:
method = op["transform"][axis]
if method == "avg":
var = cdutil.averager(var,axis="({})".format(axis))
elif method == "std":
# .std does not work with a FileVariable
# var[:] turns var into a transientVariable which can be used in .std()
var = genutil.statistics.std(var[:], axis="({})".format(axis))
else:
print("Got {} as a transform method".format(method))
if ('axis_order' in varSpec):
indexOrder = varSpec['axis_order']
axisOrder = var.getAxisIds()
stringOrder = ''.join(["({})".format(axisOrder[i]) for i in indexOrder])
var = var(order=stringOrder)
all_vars.append(var())
plot.loadVariable(all_vars)
canvas = plot.getCanvas()
windowId = self.getGlobalId(plot.getWindow())
self._canvas[windowId] = canvas
if f is not None:
f.close()
return [windowId]
def compute(dm, do):
""" Computes bias"""
if dm is None and do is None: # just want the doc
return {
"Name": "Bias",
"Abstract": "Compute Full Average of Model - Observation",
"Contact": "*****@*****.**",
}
dif = MV2.subtract(dm, do)
return MV2.float(cdutil.averager(dif, axis='xyt', weights='weighted'))
def compute(dm, do):
"""Computes bias"""
if dm is None and do is None: # just want the doc
return {
"Name": "Bias",
"Abstract": "Compute Full Average of Model - Observation",
"Contact": "*****@*****.**",
}
dif = MV2.subtract(dm, do)
return MV2.float(cdutil.averager(dif, axis="xyt", weights="weighted"))
def regional_DA(OWDA,
region,
start_time=None,
typ='fingerprint',
return_noise=False):
if start_time is None:
start_time = cdtime.comptime(2000, 1, 1)
times = np.arange(10, 76)
modeldata = mask_data(OWDA.model(region), OWDA.obs(region)[0].mask)
if typ == 'fingerprint':
mma = MV.average(modeldata, axis=0)
solver = Eof(mma, weights='area')
to_proj = mask_data(modeldata, solver.eofs()[0].mask)
P = MV.zeros(to_proj.shape[:2])
for i in range(to_proj.shape[0]):
tp = to_proj[i]
mma_mask = mask_data(mma, tp[0].mask)
solver = Eof(mma_mask, weights='area')
fac = da.get_orientation(solver)
P[i] = solver.projectField(tp)[:, 0] * fac
P.setAxisList(to_proj.getAxisList()[:2])
noise = solver.projectField(OWDA.obs(region))[:, 0]
else:
P = cdutil.averager(modeldata, axis='xy')
noise = cdutil.averager(OWDA.obs(region), axis='xy')
if return_noise:
return P, noise
else:
nmod, nyears = P.shape
TOE = MV.zeros((nmod, len(times)))
for i in range(len(times)):
L = times[i]
stop_time = start_time.add(L, cdtime.Years)
modslopes = cmip5.get_linear_trends(P(time=(start_time,
stop_time)))
noiseterm = np.std(bootstrap_slopes(noise, L))
TOE[:, i] = modslopes / noiseterm
TOE.setAxis(0, P.getAxis(0))
return TOE
def execute(self):
dataIn=self.loadData()[0]
domain = self.loadDomain()
cdms2keyargs = self.domain2cdms(domain)
f=self.loadFileFromURL(dataIn["url"])
data = f(dataIn["id"],**cdms2keyargs)
dims = "".join(["(%s)" % x for x in cdms2keyargs.keys()])
data = cdutil.averager(data,axis=dims)
data.id=dataIn["id"]
self.saveVariable(data,self.average,"json")
return
def testYxfilled(self):
f = cdms2.open(os.path.join(cdat_info.get_sampledata_path(), 'clt.nc'))
cdms2.setAutoBounds("on")
s = f("clt")
d1 = MV2.max(s, axis=0)
d2 = MV2.min(s, axis=0)
s1 = cdutil.averager(d1, axis='x')
s2 = cdutil.averager(d2, axis='x')
yf = vcsaddons.createyxvsxfill()
yf.linewidth = 5
yf.linecolor = "red"
yf.fillareacolor = "blue"
self.x.plot(s1, s2, yf)
self.checkImage("test_yxvsxfill.png")
def get_daily_ano_segment(d_seg):
"""
Note: 1. Get daily time series (3D: time and spatial 2D)
2. Meridionally average (2D: time and spatial, i.e., longitude)
3. Get anomaly by removing time mean of the segment
input
- d_seg: cdms2 data
output
- d_seg_x_ano: 2d array
"""
cdms2.setAutoBounds('on')
# sub region
d_seg = d_seg(latitude=(-10, 10))
# Get meridional average (3d (t, y, x) to 2d (t, y))
d_seg_x = cdutil.averager(d_seg, axis='y', weights='weighted')
# Get time-average in the segment on each longitude grid
d_seg_x_ave = cdutil.averager(d_seg_x, axis='t')
# Remove time mean for each segment
d_seg_x_ano = MV2.subtract(d_seg_x, d_seg_x_ave)
return d_seg_x_ano
def applyAxesOperations(self, var, axesOperations):
""" Apply axes operations to update the slab """
try:
axesOperations = eval(axesOperations)
except:
raise TypeError("Invalid string 'axesOperations'")
for axis in list(axesOperations):
if axesOperations[axis] == 'sum':
var = cdutil.averager(var, axis="(%s)" % axis, weight='equal',
action='sum')
elif axesOperations[axis] == 'avg':
var = cdutil.averager(var, axis="(%s)" % axis, weight='equal')
elif axesOperations[axis] == 'wgt':
var = cdutil.averager(var, axis="(%s)" % axis)
elif axesOperations[axis] == 'gtm':
var = genutil.statistics.geometricmean(var, axis="(%s)" % axis)
elif axesOperations[axis] == 'std':
var = genutil.statistics.std(var, axis="(%s)" % axis)
return var
def executeOperation(self, task, _input):
variable = _input.getVariable()
axis = task.metadata.get("axis","xy")
# weights = task.metadata.get( "weights", "" ).split(",")
# if weights == [""]: weights = [ ("generate" if( axis == 'y' ) else "equal") for axis in axes ]
weights = task.metadata.get("weights","generate").split(",")
if( len(weights) == 1 ): weights = weights[0]
action = task.metadata.get("action","average")
returned = 0
result_var = cdutil.averager( variable, axis=axis, weights=weights, action=action, returned=returned )
rv = self.createResult( result_var, _input, task )
return rv
def applyOperation( self, input_variables, operation ):
result = None
result_mdata = {}
t0 = time.time()
input_variable = input_variables[0]
try:
if operation is not None:
method = operation.get('method','').lower()
if method == 'average':
axis = operation.get('axis','xy').lower()
if 'e' in axis:
axis1 = axis.replace('e','')
for input_variable in input_variables:
lresult = cdutil.averager( input_variable, axis=axis1 ) if axis1 else cdutil.averager( input_variable )
result = lresult if result is None else result + lresult
result /= len( input_variables )
else:
result = cdutil.averager( input_variable, axis=axis )
except Exception, err:
wpsLog.debug( "Exception applying Operation '%s':\n %s" % ( str(operation), traceback.format_exc() ) )
def applyAxesOperations(self, var, axesOperations):
""" Apply axes operations to update the slab """
try:
axesOperations = eval(axesOperations)
except:
raise TypeError("Invalid string 'axesOperations'")
for axis in list(axesOperations):
if axesOperations[axis] == 'sum':
var = cdutil.averager(var, axis="(%s)" % axis, weight='equal',
action='sum')
elif axesOperations[axis] == 'avg':
var = cdutil.averager(var, axis="(%s)" % axis, weight='equal')
elif axesOperations[axis] == 'wgt':
var = cdutil.averager(var, axis="(%s)" % axis)
elif axesOperations[axis] == 'gtm':
var = genutil.statistics.geometricmean(var, axis="(%s)" % axis)
elif axesOperations[axis] == 'std':
var = genutil.statistics.std(var, axis="(%s)" % axis)
return var
def computer(name, start_lon, end_lon, start_lat, end_lat, Nb=200, Lb=24, windows = [20,30,40,50,75,100]):
'''
Where the hell is my docstring?
'''
# filtering parameters
fs = 1; f_hi = 1/(12*2.0); f_lo = fs/(12*7.0)
# open file
f = cdms2.open(name, 'r')
start_time = f.getAxis('time').asRelativeTime()[0]
end_time = f.getAxis('time').asRelativeTime()[-1]
# extract variable of interest in east pacific area
coral = f('pseudocoral',latitude=(start_lat,end_lat),longitude=(start_lon,end_lon))
# print 'coral'
# print coral
f.close()
# compute spatial mean
cdutil.setTimeBoundsMonthly(coral,stored=0)
spatial_mean = cdutil.averager(coral,axis='xy')
# generate boostrap samples
Xb = bootstrap.block_bootstrap_ET(spatial_mean, Lb, Nb)
#print 'spatial_mean_bootstrap'
#print spatial_mean_bootstrap
nw = len(windows) # number of windows
seasonal_amp = np.empty((nw,Nb))
variance = np.empty((nw,Nb))
index = 0 # loop over windows
for i in windows:
Xw = Xb[:,:i*12] # sample over window
clim, anom = seasonal_cycle(Xw) # isolate seasonal cycle
# compute seasonal amplitude
smax = np.nanmax(clim, axis=1)
smin = np.nanmin(clim, axis=1)
seasonal_amp[index,:] = smax - smin
# compute ENSO variance
anom2_7 = np.empty(anom.shape)
for b in range(Nb):
# apply bandpass filter
anom2_7[b,:] = bandpass.butter_bandpass_filter(anom[b,:],f_lo,f_hi,fs)
# compute variance per se
variance[index,:] = np.var(anom2_7,axis=1)
index +=1 # update index
return (variance, seasonal_amp)
def getZonalNormStd(data):
"""
data - pass filtered data
Return : normalized data and std of the meridionally averaged data.
Once we averaged over latitude, then it will become zonal data.
"""
# todo : correct the commented, docstr line for meridionally, zonally
# meridionally averaged data (averaged over latitude axis)
# weight='weighted' is default option that will generate the area weights
meridional_avg = cdutil.averager(data, axis='y')(squeeze=1) #, weight='weighted')
# make memory free
del data
# averaged over time axis
time_avg = cdutil.averager(meridional_avg, axis='t', weight='equal')(squeeze=1)
# get the std
std_avg = numpy.sqrt(meridional_avg.asma().var())
# get the normalize the data
normal_data = (meridional_avg - time_avg) / std_avg
# make memory free
del meridional_avg, time_avg
# return normal_data and std_avg data
return normal_data, std_avg
def get_zonal_cycle(tas,period=12.):
if "longitude" in tas.getAxisIds():
tas = cdutil.averager(tas,axis='x')
AMP = MV.zeros(tas.shape[1])
PHI = MV.zeros(tas.shape[1])
for j in range(tas.shape[1]):
mag,phase = get_cycle(tas[:,j],period=period)
AMP[j] = mag
PHI[j] = phase
AMP.setAxis(0,tas.getLatitude())
PHI.setAxis(0,tas.getLatitude())
return AMP,PHI
def testAnnualSeasonalAverage(self):
f = cdms2.open(self.filename, "r")
# Read in the raw data EXCLUDING a leap year
obs_timeseries1 = f('obs', time=slice(0, 48)) # 1900.1. to 1903.12.
# Read in the raw data INCLUDING a leap year
obs_timeseries2 = f('obs', time=slice(
0, 60)) # 1900.1. to 1904.12., 1904 is year lear
### Truncate first Jan, Feb and last Dec before get Annual cycle anomaly ... (to have fair DJF seasonal mean later)
obs_timeseries1 = obs_timeseries1[2:-1]
obs_timeseries2 = obs_timeseries2[2:-1]
### Set monthly time bounds ...
cdutil.setTimeBoundsMonthly(obs_timeseries1)
cdutil.setTimeBoundsMonthly(obs_timeseries2)
#### Removing Annual cycle ...
obs_timeseries_ano1 = cdutil.ANNUALCYCLE.departures(obs_timeseries1)
obs_timeseries_ano2 = cdutil.ANNUALCYCLE.departures(obs_timeseries2)
#### Calculate time average ...
obs_timeseries_ano_timeave1 = cdutil.averager(
obs_timeseries_ano1, axis='t') ## This should be zero and it does
obs_timeseries_ano_timeave2 = cdutil.averager(
obs_timeseries_ano2,
axis='t') ## This should be zero BUT it does NOT
#### SEASONAL MEAN TEST ####
obs_timeseries_ano1_DJF = cdutil.DJF(obs_timeseries_ano1,
criteriaarg=[0.95, None])
obs_timeseries_ano2_DJF = cdutil.DJF(obs_timeseries_ano2,
criteriaarg=[0.95, None])
obs_timeseries_ano1_JJA = cdutil.JJA(obs_timeseries_ano1,
criteriaarg=[0.95, None])
obs_timeseries_ano2_JJA = cdutil.JJA(obs_timeseries_ano2,
criteriaarg=[0.95, None])
#### Calculate time average ...
obs_timeseries_ano1_DJF_timeave = cdutil.averager(
obs_timeseries_ano1_DJF,
axis='t') ## This should be zero and it does
obs_timeseries_ano2_DJF_timeave = cdutil.averager(
obs_timeseries_ano2_DJF,
axis='t') ## This should be zero BUT it does NOT
obs_timeseries_ano1_JJA_timeave = cdutil.averager(
obs_timeseries_ano1_JJA,
axis='t') ## This should be zero and it does
obs_timeseries_ano2_JJA_timeave = cdutil.averager(
obs_timeseries_ano2_JJA,
axis='t') ## This should be zero and it does
numpy.testing.assert_almost_equal(obs_timeseries_ano_timeave2,
obs_timeseries_ano_timeave1, 10)
numpy.testing.assert_almost_equal(obs_timeseries_ano1_JJA_timeave,
obs_timeseries_ano2_JJA_timeave, 10)
numpy.testing.assert_almost_equal(obs_timeseries_ano1_DJF_timeave,
obs_timeseries_ano2_DJF_timeave, 10)
def reduce2lat( mv, vid=None ):
"""as reduce2lat_old, but uses the averager module for greater capabilities"""
if vid==None: # Note that the averager function returns a variable with meaningless id.
vid = 'reduced_'+mv.id
axes = allAxes( mv )
axis_names = [ a.id for a in axes if a.id!='lat' ]
axes_string = '('+')('.join(axis_names)+')'
avmv = averager( mv, axis=axes_string )
avmv.id = vid
avmv.units = mv.units
return avmv
def reduce2scalar( mv, vid=None ):
"""averages mv over the full range all axes, to a single scalar.
Uses the averager module for greater capabilities"""
if vid==None: # Note that the averager function returns a variable with meaningless id.
vid = 'reduced_'+mv.id
axes = allAxes( mv )
axis_names = [ a.id for a in axes ]
axes_string = '('+')('.join(axis_names)+')'
avmv = averager( mv, axis=axes_string )
avmv.id = vid
avmv.units = mv.units
return avmv
def executeOperation(self, task, _input):
self.logger.info( "Executing AverageKernel, input metadata = " + str(_input.metadata) )
dset_address = _input.metadata.get("uri", _input.metadata.get("dataPath") )
vname = _input.metadata.get("name")
dset = cdms2.open( dset_address )
selector = _input.getSelector( dset[vname] )
self.logger.info( "exec *EXT* AverageKernel, selector: " + str( selector ) )
variable = dset( vname, **selector )
axisIndex = variable.getAxisIndex( 'longitude' )
cdutil.times.setTimeBoundsMonthly(variable)
djfclimatology = cdutil.times.DJF.climatology(variable)
zonalAve = cdutil.averager( djfclimatology, axis=axisIndex, weights='equal' )
return self.createResult( zonalAve, _input, task )
def compute(dm, do):
""" Computes Mean Absolute Error"""
if dm is None and do is None: # just want the doc
return {
"Name": "Mean Absolute Error",
"Abstract": "Compute Full Average of " +
"Absolute Difference Between Model And Observation",
"Contact": "*****@*****.**",
}
absdif = MV2.absolute(MV2.subtract(dm, do))
mae = cdutil.averager(absdif, axis='xyt', weights='weighted')
# mae = MV.average(MV.absolute(MV.subtract(dm, do))) - depricated ... did
# not include area weights
return float(mae)
def computeTimeseries( domainId, varId, region, op ):
d = computeTimeseries.getDomain( domainId )
if d is not None:
variable = d.variables.get( varId, None )
if variable is not None:
lat, lon = region['latitude'], region['longitude']
timeseries = variable(latitude=(lat, lat, "cob"), longitude=(lon, lon, "cob"))
if op == 'average':
return cdutil.averager( timeseries, axis='t', weights='equal' ).squeeze().tolist()
else:
return timeseries.squeeze().tolist()
else:
task_error( "Missing variable '%s' in domain '%s'" % ( varId, domainId ) )
else:
task_error( "Missing domain '%s'" % ( domainId ) )
return []
def execute(data_str, domain_str, res_str):
dataIn= loadData(data_str)
domain = loadDomain(domain_str)
cdms2keyargs = domain2cdms(domain)
f=loadFileFromURL(dataIn["url"])
data = f(dataIn["id"],**cdms2keyargs)
dims = "".join(["(%s)" % x for x in cdms2keyargs.keys()])
data = cdutil.averager(data,axis=dims)
data.id=dataIn["id"]
saveVariable(data, res_str)
def reduce2latlon( mv, vid=None ):
"""as reduce2lat, but averaging reduces coordinates to (lat,lon)"""
if vid==None: # Note that the averager function returns a variable with meaningless id.
vid = 'reduced_'+mv.id
axes = allAxes( mv )
axis_names = [ a.id for a in axes if a.id!='lat' and a.id!='lon' ]
axes_string = '('+')('.join(axis_names)+')'
for ax in axes:
# The averager insists on bounds. Sometimes they don't exist, especially for obs.
if ax.id!='lat' and ax.id!='lon' and not hasattr( ax, 'bounds' ):
ax.setBounds( ax.genGenericBounds() )
avmv = averager( mv, axis=axes_string )
avmv.id = vid
avmv.units = mv.units
return avmv
