def __init__(self, dataset):
f = cdms.open("DATA/OBS/PROCESSED/" + dataset + ".nc")
self.data = {}
obs_w = f("pr_W")
self.data["west"] = obs_w
stop_time = cmip5.stop_time(obs_w)
if stop_time.month != 12:
stop_time = cdtime.comptime(stop_time.year - 1, 12, 31)
start_time = cmip5.start_time(obs_w)
if start_time.month != 1:
start_time = cdtime.comptime(start_time.year + 1, 1, 1)
obs_w = obs_w(time=(start_time, stop_time))
obs_w = fp.by_month(obs_w)
obs_e = f("pr_CE")
self.data["east"] = obs_e
stop_time = cmip5.stop_time(obs_e)
if stop_time.month != 12:
stop_time = cdtime.comptime(stop_time.year - 1, 12, 31)
start_time = cmip5.start_time(obs_e)
if start_time.month != 1:
start_time = cdtime.comptime(start_time.year + 1, 1, 1)
obs_e = obs_e(time=(start_time, stop_time))
obs_e = fp.by_month(obs_e)
self.reshaped = {}
self.reshaped["east"] = obs_e - MV.average(obs_e, axis=0)
self.reshaped["west"] = obs_w - MV.average(obs_w, axis=0)
self.reshaped["multi"] = [self.reshaped["west"], self.reshaped["east"]]
self.dataset = dataset
def standardize_soilmoisture(dataset):
f = cdms.open("../DROUGHT_ATLAS/PROCESSED/ALL_ANZDA.nc")
obs = f("pdsi")
obs = MV.masked_where(np.isnan(obs), obs)
obs = MV.masked_where(np.abs(obs) > 90, obs)
obs = obs(time=('1921-1-1', '2000-12-31'))
pdsi = mask_data(
obs, obs.mask[0]
) #Make all the obs have the same mask as the first datapoint
f.close()
mu_p = MV.average(pdsi, axis=0)
sig_p = genutil.statistics.std(pdsi, axis=0)
f = cdms.open("../DROUGHT_ATLAS/OBSERVATIONS/" + dataset +
"_soilmoisture_summerseason.nc")
gleam30cm = f("smsurf")
mask = pdsi[0].mask
gleam30cmmask = b.mask_data(gleam30cm, mask)
mu_s = MV.average(gleam30cmmask, axis=0)
sig_s = genutil.statistics.std(gleam30cmmask, axis=0)
surf = (gleam30cmmask - mu_s + mu_p) * (sig_p / sig_s)
gleam2m = f("smroot")
mask = pdsi[0].mask
gleam2mmask = b.mask_data(gleam2m, mask)
mu_s2 = MV.average(gleam2mmask, axis=0)
sig_s2 = genutil.statistics.std(gleam2mmask, axis=0)
root = (gleam2mmask - mu_s2 + mu_p) * (sig_p / sig_s)
return surf, root
def scratch(good):
#TO DO:
# mask OWDA, NADA, MADA regions
f=cdms.open("../DROUGHT_ATLAS/PROCESSED/OWDA.nc")
owda=f("pdsi")
f.close()
f = cdms.open("../DROUGHT_ATLAS/CMIP5/pdsi.ensemble.hist.rcp85.nc")
h85=f("pdsi")
good = get_rid_of_bad(h85)
owda_region = cdutil.region.domain(latitude=(np.min(owda.getLatitude()[:]),np.max(owda.getLatitude()[:])),longitude= (np.min(owda.getLongitude()[:]),np.max(owda.getLongitude()[:])))
ow = MV.average(good(owda_region),axis=0)
owsolver = Eof(MV.average(sc.mask_data(good(owda_region),owda_regrid[-1].mask),axis=0))
ow_fingerprint = owsolver.eofs()[0]
owda_regrid = owda.regrid(ow.getGrid(),regridTool='regrid2')
owda_regrid_mask=sc.mask_data(owda_regrid,ow_fingerprint.mask)
nada_region = cdutil.region.domain(latitude=(np.min(nada.getLatitude()[:]),np.max(nada.getLatitude()[:])),longitude= (np.min(nada.getLongitude()[:]),np.max(nada.getLongitude()[:])))
nasolver = Eof(MV.average(good(nada_region),axis=0))
na_fingerprint = nasolver.eofs()[0]
nada_regrid = nada.regrid(na_fingerprint.getGrid(),regridTool='regrid2')
nada_regrid_mask=sc.mask_data(nada_regrid,na_fingerprint.mask)
def average_engine(x, wts):
"""
The work horse that does the averaging! This is specific to averageing.
We will always be dealing with the first dimension.....and x is already in
the order we will average.
1-d wts are always numpy arrays or 'equal' Multi-dimensional arrays are
always MV2's
"""
#
__DEBUG__ = 0
#
if x is None: return None
if wts is None: return None
#
shx = numpy.ma.shape(x)
if __DEBUG__: print '\tInside average_engine.'
if __DEBUG__: print '\tIncoming data of shape ', shx
#
if MV2.isMaskedVariable(wts) or isinstance(wts, numpy.ndarray):
y, return_wts = MV2.average(x, weights=wts, returned=1, axis=0)
return y, return_wts
elif wts in ['equal','unweighted']:
y, return_wts = MV2.average(x, returned=1, axis=0)
return y, return_wts
else:
raise AveragerError, 'wts is an unknown type in average_engine'
return None
def average_engine(x, wts):
"""
The work horse that does the averaging! This is specific to averageing.
We will always be dealing with the first dimension.....and x is already in
the order we will average.
1-d wts are always numpy arrays or 'equal' Multi-dimensional arrays are
always MV2's
"""
#
__DEBUG__ = 0
#
if x is None: return None
if wts is None: return None
#
shx = numpy.ma.shape(x)
if __DEBUG__: print '\tInside average_engine.'
if __DEBUG__: print '\tIncoming data of shape ', shx
#
if MV2.isMaskedVariable(wts) or isinstance(wts, numpy.ndarray):
y, return_wts = MV2.average(x, weights=wts, returned=1, axis=0)
return y, return_wts
elif wts in ['equal', 'unweighted']:
y, return_wts = MV2.average(x, returned=1, axis=0)
return y, return_wts
else:
raise AveragerError, 'wts is an unknown type in average_engine'
return None
def plot_eastwest(X):
if len(X.reshaped["west"].shape) > 2:
data = [
MV.average(cmip5.ensemble2multimodel(X.reshaped["west"]), axis=0),
MV.average(cmip5.ensemble2multimodel(X.reshaped["east"]), axis=0)
]
else:
data = [X.reshaped["west"], X.reshaped["east"]]
solver = MultivariateEof(data)
weofs, eeofs = solver.eofs()
westsolver = weofs[0]
eastsolver = eeofs[0]
fac = da.get_orientation(solver)
plt.subplot(211)
months = [
"JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP", "OCT",
"NOV", "DEC"
]
plt.plot(fac * westsolver.asma(), label="WEST")
plt.plot(eastsolver.asma() * fac, label="EAST")
plt.xticks(np.arange(12), months)
plt.legend()
plt.subplot(212)
time_plot(fac * solver.pcs()[:, 0], label="WEST")
def __init__(self,data,proj="moll",typ="clim",fix_colorbar = True,**kwargs):
# matplotlib.rcParams["backend"]="TkAgg"
if data.id.find("pr")==0:
lab = "mm/day/decade"
else:
lab = "K/decade"
if len(data.shape) == 4:
self.avg = MV.average(data,axis=0)
self.data = data
else:
self.avg = data
self.data = MV.array(data.asma()[np.newaxis])
for i in range(3):
self.data.setAxis(i+1,data.getAxis(i))
if typ == "slopes":
self.plotdata= genutil.statistics.linearregression(self.avg,nointercept=1)*3650.
elif typ == "clim":
self.plotdata = MV.average(self.avg,axis=0)
elif typ == "eof":
eofdata = cdms_clone(self.avg.anom(axis=0),self.avg)
solver = Eof(eofdata)
fac = get_orientation(solver)
self.plotdata = solver.eofs()[0]*fac
if fix_colorbar:
a = max([np.abs(np.min(self.plotdata)),np.abs(np.max(self.plotdata))])
vmin = -a
vmax = a
else:
vmin=None
vmax = None
self.m = bmap(self.plotdata,alpha=1,projection=proj,vmin=vmin,vmax=vmax)
self.m.drawcoastlines()
plt.set_cmap(cm.RdBu_r)
cbar=plt.colorbar(orientation="horizontal")
cbar.set_label(lab)
self.fig = plt.gcf()
self.ax = plt.gca()
self.lat = data.getLatitude()
self.latbounds = data.getLatitude().getBounds()
self.lon = data.getLongitude()
self.cid = self.fig.canvas.mpl_connect('button_press_event',self.onclick)
self.cid2 = self.fig.canvas.mpl_connect("key_press_event",self.onpress)
self.key = "o"
self.stars = []
self.figs=[]
self.xlim = self.ax.get_xlim()
self.ylim = self.ax.get_ylim()
def temporal_mean(self):
""" Moyennes temporelles pour les champs modelises et observes """
import MV2, cdutil
import numpy as np
# --------- !!!!!!! -------- Refelchir a Mettre plutot des return pour les fonctions ------- !!!!!!! ---------
if np.ndim(self.model) < 3 :
self.model.temp_mean = self.model
self.obs.temp_mean = self.obs
else :
# Calcul de la moyenne temporelle ... Resultat => Map
self.model.temp_mean = MV2.average(self.model, axis=0)
self.obs.temp_mean = MV2.average(self.obs, axis=0)
def testAverage(self):
xav = MV2.average(self.ones, axis=1)
self.assertTrue(MV2.allequal(xav, 1))
xav2 = MV2.average(self.u_file)
xav3 = MV2.average(self.u_transient)
xav4, wav4 = MV2.average(
self.u_transient, weights=MV2.ones(
self.u_transient.shape, numpy.float), returned=1)
a = MV2.arange(5)
b = 2 ** a
av, wav = MV2.average(b, weights=a, returned=1)
self.assertEqual(av, 9.8)
self.assertEqual(wav, 10)
def temporal_mean(self):
""" Moyennes temporelles pour les champs modelises et observes """
import MV2, cdutil
import numpy as np
# --------- !!!!!!! -------- Refelchir a Mettre plutot des return pour les fonctions ------- !!!!!!! ---------
if np.ndim(self.model) < 3:
self.model.temp_mean = self.model
self.obs.temp_mean = self.obs
else:
# Calcul de la moyenne temporelle ... Resultat => Map
self.model.temp_mean = MV2.average(self.model, axis=0)
self.obs.temp_mean = MV2.average(self.obs, axis=0)
def testCrosSectionRegrid(self):
fmod = self.getDataFile(
"20160520.A_WCYCL1850.ne30_oEC.edison.alpha6_01_ANN_climo_Q.nc")
fobs = self.getDataFile("MERRA_ANN_climo_SHUM.nc")
var1 = fmod('Q')
var2 = fobs('SHUM')
mv1 = MV2.average(var1, axis=-1)
mv2 = MV2.average(var2, axis=-1)
mv1_reg = mv1
lev_out = mv1.getLevel()
lat_out = mv1.getLatitude()
mv2_reg = mv2.crossSectionRegrid(lev_out, lat_out)
self.assertTrue(numpy.ma.is_masked(mv2_reg[:, :, -1].all()))
def sum_engine(x, wts):
"""
The work horse that does the summing ! This is specific to summing.
We will always be dealing with the first dimension.....and x is already in
the order we will sum.
1-d wts are always numpy arrays or 'equal' Multi-dimensional arrays are
always MV2's
Inputs:
x : the input array (can be MV2 or MA)
wts : the input weight array (numpy array or MV2 or 'equal')
Returned:
y : The weighted sum (summed over the first dimension)
return_wts : The sum of weights (summed over the first dimension)
"""
#
__DEBUG__ = 0
#
if x is None: return None
if wts is None: return None
#
shx = numpy.ma.shape(x)
#
if __DEBUG__: print '\tInside sum_engine.'
if __DEBUG__: print '\tIncoming data of shape ', shx
#
if MV2.isMaskedVariable(wts) or isinstance(wts, numpy.ndarray):
#
# wts is an MV2 or numpy array
#
if __DEBUG__:
print '\t********** Weight is an MV2 or numpy array! **********'
#
xavg, return_wts = MV2.average(x, weights=wts, returned=1, axis=0)
y = xavg * return_wts
return y, return_wts
elif wts in ['equal', 'unweighted']:
#
# Equal weights
#
if __DEBUG__: print '\t********** Weight is Equal! **********'
xavg, return_wts = MV2.average(x, returned=1, axis=0)
y = xavg * return_wts
return y, return_wts
# end of if action == 'sum':
else:
raise AveragerError, 'wts is an unknown type in sum_engine'
return None
def plot_model_projections(fingerprint, h85):
Pwest = model_projections(fingerprint, h85, "west")
time_plot(MV.average(Pwest, axis=0),
color=get_colors("west"),
label="WEST")
Peast = model_projections(fingerprint, h85, "east")
time_plot(MV.average(Peast, axis=0),
color=get_colors("east"),
label="EAST")
Pmulti = model_projections(fingerprint, h85, "multi")
time_plot(MV.average(Pmulti, axis=0),
color=get_colors("multi"),
label="MULTI")
def sum_engine(x, wts):
"""
The work horse that does the summing ! This is specific to summing.
We will always be dealing with the first dimension.....and x is already in
the order we will sum.
1-d wts are always numpy arrays or 'equal' Multi-dimensional arrays are
always MV2's
Inputs:
x : the input array (can be MV2 or MA)
wts : the input weight array (numpy array or MV2 or 'equal')
Returned:
y : The weighted sum (summed over the first dimension)
return_wts : The sum of weights (summed over the first dimension)
"""
#
__DEBUG__ = 0
#
if x is None: return None
if wts is None: return None
#
shx = numpy.ma.shape(x)
#
if __DEBUG__: print '\tInside sum_engine.'
if __DEBUG__: print '\tIncoming data of shape ', shx
#
if MV2.isMaskedVariable(wts) or isinstance(wts, numpy.ndarray):
#
# wts is an MV2 or numpy array
#
if __DEBUG__: print '\t********** Weight is an MV2 or numpy array! **********'
#
xavg, return_wts = MV2.average(x, weights=wts, returned=1, axis=0)
y = xavg * return_wts
return y, return_wts
elif wts in ['equal','unweighted']:
#
# Equal weights
#
if __DEBUG__: print '\t********** Weight is Equal! **********'
xavg, return_wts = MV2.average(x, returned=1, axis=0)
y = xavg * return_wts
return y, return_wts
# end of if action == 'sum':
else:
raise AveragerError, 'wts is an unknown type in sum_engine'
return None
def get_ohc(forcing,
average=True,
prefix="/Users/kmarvel/Google Drive/ECS/OHC_DATA/"):
""" read in the ocean heat content"""
direc = prefix + forcing + "/"
files = sorted(glob.glob(direc + "*.nc"))
L = len(files)
f = cdms.open(files[0])
ohc_raw = f("ohc")
tax = ohc_raw.getTime()
ohc_sum = MV.sum(ohc_raw, axis=1)
ohc_final = (ohc_sum) * 1.e-22 #Convert to 1e22 Joules
nt = len(ohc_final)
MMA = MV.zeros((L, nt))
MMA[0] = ohc_final
f.close()
#Loop over all files in directory (ensemble members)
for i in range(L)[1:]:
f = cdms.open(files[i])
ohc_raw = f("ohc")
ohc_sum = MV.sum(ohc_raw, axis=1)
ohc_final = (ohc_sum) * 1.e-22
MMA[i] = ohc_final
f.close()
#Return multilodel average, or not
if average:
MMA = MV.average(MMA, axis=0)
MMA.setAxis(0, tax)
else:
MMA.setAxis(1, tax)
return MMA
def get_tas(forcing, average=True):
""" Read in pre-computed annual-average, global-average surface air temperatures (NOT anomalies)"""
if forcing == 'GHG':
forcing = 'historicalGHG'
model = "GISS-E2-R"
p = "*p1*"
direc = "/Users/kmarvel/Google Drive/HistoricalMisc/GLOBAL_MEAN/" + forcing + "/"
files = get_ensemble(direc, model, search_string="*p1*YEAR*")
f = cdms.open(files[0])
c = 0
f = cdms.open(files[0])
start = f["tas"].getTime().asComponentTime()[0]
stop = f["tas"].getTime().asComponentTime()[-1]
tax = f("tas").getTime()
L = len(f("tas", time=(start, stop)))
tas = MV.zeros((len(files), L))
f.close()
for fil in files:
f = cdms.open(fil)
tas[c] = f("tas", time=(start, stop))
c += 1
if average:
temperatures = MV.average(tas, axis=0)
temperatures.setAxis(0, tax)
else:
temperatures = tas
temperatures.setAxis(1, tax)
return temperatures
def onclick(self, event):
if not event.inaxes:
return
#if True:
x = event.xdata
y = event.ydata
lon, lat = self.m(x, y, inverse=True)
if self.m.lonmin > -100.:
if lon < 0:
lon = 360 + lon
xy = (lat, lon)
t = get_plottable_time(self.data)
i, j = find_ij(self.data, xy)
X, Y = self.m(self.lon[j], self.lat[i])
self.stars += [self.m.plot(X, Y, "y*", markersize=15)[0]]
f2 = plt.figure()
self.figs += [f2]
ax2 = f2.add_subplot(111)
plt.draw()
for mod in range(self.data.shape[0]):
ax2.plot(t, self.data[mod, :, i, j].asma(), color=cm.gray(.5))
ax2.plot(t, MV.average(self.data, axis=0)[:, i, j].asma(), color="k")
ax2.set_title("(" + str(self.lat[i]) + "," + str(self.lon[j]) + ")")
plt.draw()
def get_crossing_time(region, variable, scenario, month):
vcert = stats.norm.interval(.99)[1]
mfile = get_file(region, variable, scenario, month)
if mfile is None:
crossing_time = None
else:
f = cdms.open(mfile)
data = f(variable + "_SN")
avg = MV.average(data, axis=0)
threshexceed = np.where(np.abs(avg) > vcert)[0]
#If it never exceeds the threshold, return None
if len(threshexceed) == 0:
return (None)
#If it hasn't exceeded the threshold by the last time step, return None
if len(avg) - 1 not in threshexceed:
return (None)
if len(np.where(np.diff(threshexceed) > 1)[0]) > 0:
isnot1 = np.max(np.where(np.diff(threshexceed) > 1)[0]) + 1
else:
isnot1 = 0
staysabove = int(threshexceed[isnot1])
crossing_time = int(cmip5.get_plottable_time(data)[staysabove])
f.close()
return (crossing_time)
def dictionary_ensemble_average(d, grid=None):
if grid is None:
shape = 1.e20
if grid is None:
for m in d.keys():
gridsize = d[m].shape[-1] * d[m].shape[-2]
if gridsize < shape:
shape = gridsize
themodel = m
coarsest_grid = d[themodel].getGrid()
allstop = str(np.min([cmip5.stop_time(d[m]).year
for m in d.keys()])) + "-12-31"
allstart = str(np.max([cmip5.start_time(d[m]).year
for m in d.keys()])) + "-1-11"
standardize = lambda data: data(time=(allstart, allstop)).regrid(
coarsest_grid, regridTool='regrid2')
counter = 0
goodmodels = list(d)
L = len(goodmodels)
for m in d.keys():
modeldata = standardize(MV.average(
d[m], axis=0)) # average over individual ensemble members
if counter == 0:
MME = MV.zeros((L, ) + modeldata.shape)
MME[counter] = modeldata
counter += 1
modax = cmip5.make_model_axis(list(d))
axlist = [modax] + modeldata.getAxisList()
MME.setAxisList(axlist)
cdutil.setTimeBoundsMonthly(MME)
#MME.id=variable
return MME
def truncated_solver(D,the_start=None,the_stop=None,include_cru=False,include_dai=False):
thedata = D.ALL.model(time=(the_start,the_stop))
the_mma=MV.average(cmip5.ensemble2multimodel(thedata),axis=0)
if include_cru:
f = cdms.open("../DROUGHT_ATLAS/OBSERVATIONS/CRU_selfcalibrated.nc")
cru_jja=f("pdsi")
f.close()
cru_jja_mask = mask_data(cru_jja,D.ALL.obs[0].mask)(time=(the_start,'2018-12-31'))
newmask = np.prod(~cru_jja_mask.mask,axis=0)
cru_jja_mask = mask_data(cru_jja_mask,newmask==0)
thesolver = Eof(mask_data(D.ALL.mma(time=(the_start,the_stop)),newmask==0),weights='area')
elif include_dai:
f = cdms.open("../DROUGHT_ATLAS/OBSERVATIONS/DAI_selfcalibrated.nc")
dai_jja=f("pdsi")
f.close()
dai_jja_mask = mask_data(dai_jja,D.ALL.obs[0].mask)(time=(the_start,'2018-12-31'))
newmask = np.prod(~dai_jja_mask.mask,axis=0)
dai_jja_mask = mask_data(dai_jja_mask,newmask==0)
thesolver = Eof(mask_data(D.ALL.mma(time=(the_start,the_stop)),newmask==0),weights='area')
else:
thesolver = Eof(the_mma,weights="area")
return thesolver
def soilmoisture_fingerprints(mask,name=None):
depths = ["30cm","2m","pdsi"]
letters = ["(a): ","(b): ","(c): "]
pcs = []
pclabels = []
for depth in depths:
i=depths.index(depth)
plt.subplot(2,2,i+1)
sm = soilmoisture(depth,mask=mask)
solver = Eof(MV.average(sm,axis=0))
fac = da.get_orientation(solver)
if name is None:
m=landplot(fac*solver.eofs()[0],vmin=-.1,vmax=.1)
plt.colorbar(orientation='horizontal',label='EOF loading')
else:
m=plot_regional(fac*solver.eofs()[0],name,vmin=-.1,vmax=.1)
m.drawcountries()
m.drawcoastlines(color='gray')
plt.title(letters[i]+depth+" fingerprint")
pcs+=[fac*solver.pcs()[:,0]]
plt.subplot(2,2,4)
for i in range(3):
time_plot(pcs[i],label=depths[i])
plt.legend(loc=0)
plt.title("(d): Principal Components")
plt.xlabel("Time")
plt.ylabel("Temporal amplitude")
def compare_pre_post_1100_noise(X,L=31,latbounds=None):
time1=('1100-1-1','1399-12-31')
c1=cm.Purples(.8)
time2=('1400-1-1','2005-12-31')
if latbounds is not None:
obs=X.obs(latitude=latbounds)
mma = MV.average(X.model(latitude=latbounds),axis=0)
mma = mask_data(mma,obs[0].mask)
solver = Eof(mma)
obs = mask_data(obs,solver.eofs()[0].mask)
truncnoise=solver.projectField(obs)[:,0]*da.get_orientation(solver)
noisy1=truncnoise(time=time1)
noisy2=truncnoise(time=time2)
else:
noisy1=X.noise(time=time1)
noisy2=X.noise(time=time2)
c2=cm.viridis(.1)
plt.subplot(121)
Plotting.Plotting.time_plot(noisy1,c=c1)
Plotting.Plotting.time_plot(noisy2,c=c2)
plt.ylabel("Projection")
plt.title("(a): Noise time series")
plt.subplot(122)
plt.hist(b.bootstrap_slopes(noisy1,L),color=c1,normed=True,alpha=.5)
da.fit_normals_to_data(b.bootstrap_slopes(noisy1,L),c=c1,label="1100-1400")
plt.hist(b.bootstrap_slopes(noisy2,L),color=c2,normed=True,alpha=.5)
da.fit_normals_to_data(b.bootstrap_slopes(noisy2,L),c=c2,label="1400-2005")
plt.legend()
plt.title("(b): 31-year trend distributions")
return np.std(b.bootstrap_slopes(noisy1,L)),np.std(b.bootstrap_slopes(noisy2,L))
def pad_by_10(X, year1, year2):
"""
Pad an array at the beginning with an artificial 10 year spinup, in which each value is set to the climatology
"""
if type(year1) == type("string"):
year, month, day = year1.split("-")
year1 = cdtime.comptime(int(year), int(month), int(day))
if type(year2) == type("string"):
year, month, day = year2.split("-")
year2 = cdtime.comptime(int(year), int(month), int(day))
tax = X.getTime()
lastten = [year1.sub(x, cdtime.Months) for x in range(121)[1:]][::-1]
dayax = np.array([x.torel(tax.units).value for x in lastten])
tax_new = np.append(dayax, tax)
new_time_axis = cdms.createAxis(tax_new)
new_time_axis.designateTime()
new_time_axis.id = "time"
new_time_axis.units = tax.units
Xnew = MV.zeros((len(tax_new), ) + X.shape[1:])
Xnew[:120] = np.repeat(MV.average(X(time=(year1, year2)),
axis=0).asma()[np.newaxis, :, :],
120,
axis=0)
Xnew[120:] = X
for k in X.attributes.keys():
setattr(Xnew, k, X.attributes[k])
Xnew.setAxisList([new_time_axis] + X.getAxisList()[1:])
return Xnew
def plot_seasonal_trends(self, ax=None, scenario="ssp585"):
if ax is None:
fig = plt.figure()
ax = plt.subplot(111)
if scenario.find("+") > 0:
ssp = scenario.split("+")[-1]
test = self.splice_historical(ssp)
else:
test = self.ensemble_average(scenario)
#test=self.ensemble_average("ssp585")
mma = MV.average(test, axis=0)
nyears = int(len(mma) / 12)
tst = mma.reshape((nyears, 12))
tmp = [
ax.plot(tst.asma()[i], c=cm.magma(i / float(nyears)))
for i in range(nyears)
]
months = [
"JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP",
"OCT", "NOV", "DEC"
]
ax.set_xticks(np.arange(12))
ax.set_xticklabels(months)
ax.set_ylabel(self.variable)
ax.set_title(self.region)
def testReshapeMaskedAverage(self):
a = MV2.arange(100)
a = MV2.reshape(a, (10, 10))
self.assertEqual(a.shape, (10, 10))
self.assertEqual(len(a.getAxisList()), 2)
a = MV2.masked_greater(a, 23)
b = MV2.average(a, axis=0)
c = a - b
def plot_model_trends(self, i, start=None, stop=None):
if i != "avg":
west = self.h85.reshaped["west"][i]
east = self.h85.reshaped["east"][i]
else:
west = MV.average(h85.reshaped["west"], axis=0)
east = MV.average(h85.reshaped["east"], axis=0)
if start is not None:
west = west(time=(start, stop))
east = east(time=(start, stop))
plt.plot(cmip5.get_linear_trends(west).asma(), label="WEST")
plt.plot(cmip5.get_linear_trends(east).asma(), label="EAST")
months = [
"JAN", "FEB", "MAR", "APR", "MAY", "JUN", "JUL", "AUG", "SEP",
"OCT", "NOV", "DEC"
]
plt.xticks(np.arange(12), months)
def rcp_solver(D,early_start=None,early_stop=None):
if early_start is None:
early_start = cdtime.comptime(2006,1,1)
if early_stop is None:
early_stop=cdtime.comptime(2099,12,31)
earlydata = D.ALL.model(time=(early_start,early_stop))
early_mma=MV.average(cmip5.ensemble2multimodel(earlydata),axis=0)
earlysolver = Eof(early_mma,weights="area")
return earlysolver
def main():
# mip = 'cmip5'
mip = 'cmip6'
exp = 'historical'
version = 'v20200625'
period = '1985-2004'
datadir = '/p/user_pub/pmp/pmp_results/pmp_v1.1.2/diagnostic_results/mjo/' + mip + '/historical/' + version
imgdir = '/p/user_pub/pmp/pmp_results/pmp_v1.1.2/graphics/mjo/' + mip + '/historical/' + version
ncfile_list = glob.glob(os.path.join(datadir, '*.nc'))
# get list of models
models_list = sorted(
[r.split('/')[-1].split('.')[0].split('_')[1] for r in ncfile_list])
# remove repeat
models_list = list(dict.fromkeys(models_list))
# remove obs
models_list.remove('obs')
# models_list = models_list[0:1]
print(models_list)
for model in models_list:
ncfile_list_model = glob.glob(
os.path.join(datadir, '*' + model + '*.nc'))
runs_list = sorted([
r.split('/')[-1].split('.')[0].split('_')[3]
for r in ncfile_list_model
])
print(model, runs_list)
d_runs = []
for run in runs_list:
try:
ncfile = '_'.join(
[mip, model, exp, run, 'mjo', period, 'cmmGrid']) + '.nc'
f = cdms2.open(os.path.join(datadir, ncfile))
d = f('power')
d_runs.append(d)
f.close()
except Exception as err:
print(model, run, 'cannnot load:', err)
pass
if run == runs_list[-1]:
num_runs = len(d_runs)
# ensemble mean
d_avg = MV2.average(d_runs, axis=0)
d_avg.setAxisList(d.getAxisList())
title = (mip.upper() + ': ' + model + ' (' + str(num_runs) +
' runs mean) \n Pr, NDJFMA, ' + period +
', common grid (2.5x2.5deg)')
# E/W ratio
ewr, eastPower, westPower = calculate_ewr(d_avg)
# plot prepare
pngfilename = ncfile.split('.nc')[0].replace(run, 'average')
fout = os.path.join(imgdir, pngfilename)
# plot
plot_power(d_avg, title, fout, ewr)
def aerosol_solver(D,aerosol_start=None,aerosol_stop=None,include_cru=False):
if aerosol_start is None:
aerosol_start = cdtime.comptime(1950,1,1)
if aerosol_stop is None:
aerosol_stop=cdtime.comptime(1975,1,1)
aerosoldata = D.ALL.model(time=(aerosol_start,aerosol_stop))
aerosol_mma=MV.average(cmip5.ensemble2multimodel(aerosoldata),axis=0)
aerosolsolver = Eof(aerosol_mma,weights="area")
return aerosolsolver
def plot_future(self,season,single_member=True,plot_historical=True):
if single_member:
func=self.single_member_ensemble
else:
func=self.ensemble_average
hist=func("historical")
start_time=cdtime.comptime(1951,1,1)
end_time=cdtime.comptime(1980,12,31)
clim = getattr(cdutil,season).climatology(hist(time= (start_time, end_time, 'co')))
if plot_historical:
timedata=getattr(cdutil,season).departures(hist,ref=clim)
Plotting.time_plot(MV.average(timedata,axis=0),color=get_color("historical"),label=get_label("historical"))
for ssp in ["ssp126","ssp245","ssp370","ssp585"]:
rawdata=func(ssp)
timedata=getattr(cdutil,season).departures(rawdata,ref=clim)
Plotting.time_plot(MV.average(timedata,axis=0),color=get_color(ssp),label=get_label(ssp))
plt.legend()
def compute(dm, do):
""" Computes bias"""
if dm is None and do is None: # just want the doc
return {
"Name": "Bias",
"Abstract": "Compute Full Average of Model - Observation",
"Contact": "Peter Gleckler <*****@*****.**>",
}
return MV2.float(MV2.average(MV2.subtract(dm, do)))
def jja_pdsi(X):
"""Average CMIP5 PDSI over boreal summer months """
jja = MV.average(X[:, 5:8], axis=1)
tax = jja.getAxis(0)
tax.units = 'years since 0000-7-1'
tax.id = "time"
jja.setAxis(0, tax)
jja.id = 'pdsi'
return jja
def multi_model_ensemble(variable,
experiment,
model_ensemble_average=False,
models=None,
grid=None):
rawdir = get_rawdir(variable)
if experiment.find("hist") >= 0:
time_bounds = ('1850-1-1', '2014-12-31')
elif experiment.find("ssp") >= 0:
time_bounds = ('2015-1-1', '2100-12-31')
tax = get_tax(experiment)
L = int(len(tax) / 12)
if models is None:
models = cmip6_models_with_all_variables([variable], experiment)
if grid is None:
model = models[0]
rip = get_rips(model, variable, experiment)[0]
allfiles = sorted(
glob.glob(rawdir + model + "/*." + experiment + ".*." + rip +
".*"))
f = cdms.open(allfiles[0])
data = f(variable)
#for model in get_ok_models("SW"):
grid = data.getGrid()
f.close()
nmod = len(models)
MME = MV.zeros((nmod, L * 12) + grid.shape) + 1.e20
for i in range(nmod):
print(models[i])
model = models[i]
if model_ensemble_average:
ens = get_ensemble(model, variable, experiment)
data = MV.average(ens, axis=0)
else:
data = get_firstmember(model, variable, experiment)
data_regrid = data.regrid(grid, regridTool='regrid2')(time=time_bounds)
if data_regrid.shape == MME[i].shape:
MME[i] = data_regrid
else:
print("problem with", model)
MME = MV.masked_where(MME > 1.e10, MME)
modax = cmip5.make_model_axis(models)
lat = grid.getLatitude()
lon = grid.getLongitude()
axlist = [modax, tax, lat, lon]
MME.setAxisList(axlist)
MME.id = variable
return MME
def compute(dm, do):
""" Computes Mean Absolute Error"""
if dm is None and do is None: # just want the doc
return {
"Name": "Mean Absolute Error",
"Abstract": "Compute Full Average of " +
"Absolute Difference Between Model And Observation",
"Contact": "Peter Gleckler <*****@*****.**>",
}
mae = MV.average(MV.absolute(MV.subtract(dm, do)))
return float(mae)
def describe(obj, stats=None, format=pprint.pformat):
'''Return the object decription depending on its type.
Usefull with numpy and cdms variables/axes
:Params:
- **obj**: The object to describe.
- **stats**: If True, include numerical information like min, max, mean, count, ...
:Return:
- The object's summary string
'''
try:
import cdms2, MV2, numpy
from cdms2.avariable import AbstractVariable
from cdms2.axis import AbstractAxis
if isinstance(obj, configobj.ConfigObj):
obj = obj.dict()
if not isinstance(obj, (cdms2.avariable.AbstractVariable, cdms2.axis.AbstractAxis, numpy.ndarray)):
return format(obj)
otype = obj.__class__.__name__ # type(obj)
sh, sz = MV2.shape(obj), MV2.size(obj)
mi = ma = av = co = None
if stats and sz:
try:
if hasattr(obj, 'typecode') and obj.typecode() not in ('c',):
mi, ma, av, co = MV2.min(obj), MV2.max(obj), MV2.average(obj), MV2.count(obj)
except: logger.exception('Error getting statistics of object %s', type(obj))
if isinstance(obj, AbstractVariable):
return '%s: %s, shape: (%s), order: %s%s'%(
otype, obj.id,
','.join('%s=%s'%(a.id, a.shape[0]) for a in obj.getAxisList()),
obj.getOrder(),
stats and ', min: %s, max: %s, avg: %s, count: %s'%(mi, ma, av, co) or '')
elif isinstance(obj, AbstractAxis):
return '%s: %s, size: %s%s'%(
otype, obj.id, sz,
stats and ', min: %s, max: %s, avg: %s, count: %s'%(mi, ma, av, co) or '')
else:#if isinstance(obj, numpy.ndarray):
return '%s: shape: %s%s'%(
otype, sh,
stats and ', min: %s, max: %s, avg: %s, count: %s'%(mi, ma, av, co) or '')
except Exception, e:
logger.exception('Error getting description of object %s', type(obj))
return '%s (error getting description)'%(type(obj))
def process(self,data):
## if self.symetric:
## data = symetric(data)
# Make sure we have an even number of time steps
t=data.getTime()
# length of time axis
nt=len(t)
if nt%2!=0:
print "Warning time wasn't even, removed last time step"
data=data[:-1]
t=data.getTime() ## get the new time axis
nt=len(t)
if len(t)<self._NTSub:
raise Exception,"Error your data must have at least %i time steps, adjust frequency (currently: %i/day) or number_of_days (currently: %i processed at once) to reach that limit, or get more data" % (self._NTSub,self.frequency,self.number_of_days)
## Computes PP, number of sub-domain
PP=float(nt-self._NTSub)/self._NShift+1
PP=int(PP)
## Number of longitudes
lons=data.getLongitude()
NL=len(lons)
tt=cdms2.createAxis(numpy.arange(self._NTSub),id='sub_time')
## Should redo that with just an arange (eventually...)!!!
## Frequencies in cycles/day
ff=numpy.arange(0,self._NTSub+1,1,numpy.float)
for i in range(1,self._NTSub+2):
ff[i-1]=float(i-1-self._NTSub/2.)*self.frequency/float(self._NTSub)
## Should redo that with just an arange (eventually...)!!!
## Wave numbers
ss=numpy.arange(0,NL+1,1,numpy.float)
for i in range(1,NL+2):
ss[i-1]=float(i-1-NL/2.)
## print 'Frequencies:',ff
## print 'Wave numbers:',ss
## Ok, we now do the real stuff
## Creates the array of powers (Number of subtimes,fqcy,wave_numbers,latitudes)
lats=data.getLatitude()
Power=numpy.zeros((PP,self._NTSub+1,NL+1,len(lats)),numpy.float)
## LOOP through time sub domains
prev=0 # initialize the scrolling bar
for Pcount in range(PP):
if PP>1:
prev=genutil.statusbar(float(Pcount),PP-1,prev=prev,tk=self.tkbar)
## Get the time subdomain
EEo=data[Pcount*self._NShift:Pcount*self._NShift+self._NTSub](order='tx...')
## First does the symetric/antisymetric thing if needed
if self.symetric: EEo=symetrick(EEo)
## Now detrending
## Step 1- Get the slope and intercept
slope,intercept=genutil.statistics.linearregression(EEo,nointercept=0)
## Step 2- remove the trend
## Step 2a: Create an array with the time values
a=EEo.getTime()
A=MV2.array(a[:],typecode='d')
A.setAxis(0,a)
## Step 2b: "Grows" it so it has the same shape than data
A,EEo=genutil.grower(A,EEo)
## Step 2c: Actually remove the trend
EE=EEo-A*slope-intercept
## we don't need A,EEo,slope,intercept anymore
del(EEo)
del(slope)
del(intercept)
del(A)
## Remove the time mean
mean=MV2.average(EE,0)
EE=EE-mean
del(mean) # could be big in memory
## Tapering time...
tapertozero(EE,1,len(EE)-1,5*self.frequency)
## OK here Wheeler has some windowing on longitude, but it's commented out
## I'll pass it for now
## Ok the actuall FFT work
EE=numpy.fft.fft2(EE,axes=(1,0))/NL/self._NTSub
## OK NOW THE LITTLE MAGIC WITH REORDERING !
A=numpy.absolute(EE[0:self._NTSub/2+1,1:NL/2+1])**2
B=numpy.absolute(EE[self._NTSub/2:self._NTSub,1:NL/2+1])**2
C=numpy.absolute(EE[self._NTSub/2:self._NTSub,0:NL/2+1])**2
D=numpy.absolute(EE[0:self._NTSub/2+1,0:NL/2+1])**2
Power[Pcount,self._NTSub/2:,:NL/2]=A[:,::-1]
Power[Pcount,:self._NTSub/2,:NL/2]=B[:,::-1]
Power[Pcount,self._NTSub/2+1:,NL/2:]=C[::-1,:]
Power[Pcount,:self._NTSub/2+1,NL/2:]=D[::-1,:]
## End of Pcount loop
if self.tkbar and PP>1:
prev[1].destroy()
prev[0].destroy()
## Now generates the decorations
## first the time axis (subdomains)
vals=[]
bounds=[]
pp=0
for i in range(0,len(t)-self._NShift,self._NShift):
st=t.subAxis(i,i+self._NTSub)
if len(st[:])==self._NTSub:
pp+=1
vals.append((st[0]+st[-1])/2.)
bds=st.getBounds()
#print 'Bounds:',bds
if bds is None:
raise ValueError, "Data need to have bounds on time dimension"
else:
bounds.append([bds[0][0],bds[-1][1]])
## Convert lists to arrays
vals=numpy.array(vals)
bounds=numpy.array(bounds)
## Creates the time axis
dumt=cdms2.createAxis(vals,bounds=bounds)
dumt.id='time'
dumt.units=t.units
dumt.designateTime()
dumt.setCalendar(t.getCalendar())
## Create the frequencies axis
T=cdms2.createAxis(ff)
T.id='frequency'
T.units='cycles per day'
## Create the wave numbers axis
S=cdms2.createAxis(ss)
S.id='planetaryzonalwavenumber'
S.units='-'
## Makes it an MV2 with axis and id (id come sfrom orignal data id)
Power=MV2.array(Power,axes=(dumt,T,S,lats),id=data.id+'_'+'power')
## Adds a long name attribute
Power.longname='Real power spectrum for the many different parts (i.e. over separate time divisions)'
## And return the whole thing ordered 'time', 'latitude', 'frequencies','wavenumbers'
return Power(order='ty...')
print absd #******************************************************************************* # # Example 2: # In this example, we compute the spatial correlation between 2 fields # - say the mean surface air temperature for the 1980-1989 period # versus each month of the 1980-1989 period. # #******************************************************************************* # # First compute the average for the 1980-1990 period (designated ncep2 above) # mncep2 = MV2.average(ncep2) # # And now correlate this pattern with each month of ncep2 # cor = statistics.correlation(ncep2, mncep2, axis='xy') print 'Correlation:' print cor #******************************************************************************* # # Example 3: # We take the correlation example above a step further and specify # that the correlation be computed with weights based on latitude # area taken into account. Note that the correlation can also be
# get the file id from the call to cdat open file through the cdms2 module fid = cdms2.open(cmbe_file) #------------------ # read some variable to get time p_sfc=fid( 'p_sfc', squeeze = 0, order = '0' ) cdutil.times.setTimeBoundsDaily(p_sfc,24) # set time and time_bounds time = p_sfc.getTime() time_bounds = time.getBounds() time_units = time.units # to check some statistics on the data vv = fid['p_sfc'] 'var name = ',vv.id 'average = ',MV2.average(vv,axis=0).data 'min = ',MV2.minimum(vv) 'max = ',MV2.maximum(vv) #------------------ # get longitude and latitude lon = fid['lon'] # in memory variable - force it to be a cdms variable lat = fid['lat'] # need to change manually degrees_west to degrees_east, which is required by cmor table lon_units = lon.units if lon_units == 'degrees_west': lon = -lon() # change it to be a scalar value by applying () else: lon = lon() # need to change manually degrees_south to degrees_north which is required by cmor table
temporary[region] = d_sub_aave(
time=(start_t, end_t))
d_sub_aave = MV2.concatenate(
[part1, part2], axis=0)
if debug:
print('debug: ', region, year,
d_sub_aave.getTime().asComponentTime())
# get pentad time series
list_d_sub_aave_chunks = list(
divide_chunks_advanced(d_sub_aave, n, debug=debug))
pentad_time_series = []
for d_sub_aave_chunk in list_d_sub_aave_chunks:
# ignore when chunk length is shorter than defined
if d_sub_aave_chunk.shape[0] >= n:
ave_chunk = MV2.average(
d_sub_aave_chunk, axis=0)
pentad_time_series.append(float(ave_chunk))
if debug:
print('debug: pentad_time_series length: ',
len(pentad_time_series))
# Keep pentad time series length in consistent
ref_length = int(365/n)
if len(pentad_time_series) < ref_length:
pentad_time_series = interp1d(
pentad_time_series, ref_length, debug=debug)
pentad_time_series = MV2.array(pentad_time_series)
pentad_time_series.units = d.units
pentad_time_series_cumsum = np.cumsum(
pentad_time_series)
def plot_ped_mod_on_pro(self, model, profiles, select=None, **kwargs):
'''Plot potential energy deficit of model data correspponding to profiles position
:Params: See: :func:`coloc_mod_on_pro`
'''
self.verbose('Plotting PED of colocalized %s on %s\nselect: %s\nkwargs: %s',
model.__class__.__name__, profiles.__class__.__name__, select, kwargs)
# =====
# Lecture des donnees de stratification
lons_mod, lats_mod, deps_mod, temp_mod, sal_mod, pres_mod, dens_mod = \
self.coloc_strat_mod_on_pro(model, profiles, select, **kwargs)
# =====
# Calcul ped modele
ddeps = meshweights(deps_mod, axis=0)
# Densité moyenne
dmean = MV2.average(dens_mod, axis=0, weights=ddeps)
# Anomalie de densité
danom = dens_mod-dmean
# Énergie potentielle disponible
ape = danom * g
ape *= ddeps
# Deficit
ped_mod = MV2.average(ape, axis=0, weights=ddeps)
ped_mod.units = 'J.m^{-2}'
ped_mod.long_name = u"Definit d'energie potentielle"
ped_mod.setAxisList(lons_mod.getAxisList())
model.verbose('PED: %s', ped_mod)
# =====
# Calcul ped profiles
ped_pro, lats_pro, lons_pro = profiles.get_ped(select)
profiles.verbose('PED: %s', ped_pro)
# =====
# Tracés
vmin = ped_pro.min()
vmax = ped_pro.max()
if ped_mod.count():
vmin = min(ped_mod.min(), vmin)
vmax = max(ped_mod.max(), vmax)
else:
self.warning('No model data')
if 'latitude' in select:
lat_min, lat_max = select['latitude'][:2]
else:
la = model.get_latitude()
lat_min, lat_max = min(la), max(la)
if 'longitude' in select:
lon_min, lon_max = select['longitude'][:2]
else:
lo = model.get_longitude()
lon_min, lon_max = min(lo), max(lo)
levels = auto_scale((vmin, vmax))
vmin = levels[0]
vmax = levels[-1]
# Création de la palette
cmap = cmap_magic(levels)
# On trace de champ du modèle moyené sur la période choisie
m = map2(lon=(lon_min, lon_max), lat=(lat_min, lat_max), show=False)
# On trace le modèle en fond
msca = None
try: msca = m.map.scatter(lons_mod, lats_mod, s=100, c=ped_mod, vmin=vmin, vmax=vmax, cmap=cmap, label='model')
except: self.exception('Failed plotting model data')
# On trace les observations avec des points plus petits
psca = None
try: psca = m.map.scatter(lons_pro, lats_pro, s=25, c=ped_pro, vmin=m.vmin, vmax=m.vmax, cmap=m.cmap, label='profiles')
except: self.exception('Failed plotting profiles data')
# Colorbar
if msca is not None:
colorbar(msca)
elif psca is not None:
colorbar(psca)
print ac1
data1=cdutil.ANNUALCYCLE.departures(data1,ref=ac1)
data2=cdutil.ANNUALCYCLE.departures(data2,ref=ac2)
print data1.shape,data2.shape
tim = data2.getTime()
lat=data2.getLatitude()
lon=data2.getLongitude()
data1=cdms2.createVariable(data1,axes=[tim,lat,lon],typecode='f',id='tas')
diff=MV2.subtract(data1,data2)
# zonal differences
z_diff=MV2.average(diff,2)
print 'Zonal data shape (before): ',z_diff.shape
z_diff=MV2.transpose(z_diff,(1,0))
# add id to data
z_diff.id='zonal_diff'
print 'Zonal data shape (after): ',z_diff.shape
# global differences
gl_diff=cdutil.averager(diff,axis='xy')
x=vcs.init()
x.setcolormap('default')
fill=x.getisofill('default')
x.plot(z_diff,fill)
]
for file in files:
f=cdms.open(file)
u=f('u')
if file == files[0]: # First file
sh=list(u.shape) # Create a list with the shape of the data
sh.insert(0,1) # Insert value 1 in front of the list
accumulation = u
newdim = MV.reshape(u,sh) # Create a new 1D dimension
else:
# add u at the end of accumaltion on dimension 0
accumulation = MV.concatenate((accumulation,u))
tmp = MV.reshape(u,sh) # Create a new 1D dimension
newdim = MV.concatenate((newdim,tmp)) # Add u to the newdim over the new dimension
f.close()
print accumulation.shape # All time added over the same dimension
print newdim.shape # Has a new dimension for years
avg = MV.average(accumulation)
std = genutil.statistics.std(newdim)
print avg.shape
print std.shape
import MV2
from markError import clearError,markError,reportError
print 'Test 15: reshape and mask and average ...',
a=MV2.arange(100)
try:
failed = False
a.shape=(10,10)
except:
failed = True
a = MV2.reshape(a,(10,10))
if failed is True: markError('shape should not have worked (protected attribute)')
if len(a.getAxisList())!=2: markError('reshape did not produce 2 axes')
a=MV2.masked_greater(a,23)
b=MV2.average(a,axis=0)
c=a-b
def compute(dm,do):
""" Computes Mean Absolute Error"""
mae = MV.average(MV.absolute(MV.subtract(dm,do)))
return float(mae)
def compute(dm,do):
""" Computes bias"""
return MV.float(MV.average(MV.subtract(dm,do)))
def compute(params):
fileName = params.fileName
month = params.args.month
monthname = params.monthname
varbname = params.varname
template = populateStringConstructor(args.filename_template, args)
template.variable = varbname
# Units on output (*may be converted below from the units of input*)
outunits = 'mm/d'
startime = 1.5 # GMT value for starting time-of-day
reverted = template.reverse(os.path.basename(fileName))
dataname = reverted["model"]
if dataname not in args.skip:
try:
print('Data source:', dataname)
print('Opening %s ...' % fileName)
f = cdms2.open(fileName)
# Composite-mean and composite-s.d diurnal cycle for month and year(s):
iYear = 0
for year in range(args.firstyear, args.lastyear + 1):
print('Year %s:' % year)
startTime = cdtime.comptime(year, month, 1, 1, 30)
# Last possible second to get all tpoints
finishtime = startTime.add(
1, cdtime.Month).add(-1.5, cdtime.Hour).add(.1, cdtime.Second)
print('Reading %s from %s for time interval %s to %s ...' % (varbname, fileName, startTime, finishtime))
# Transient variable stores data for current year's month.
tvarb = f(varbname, time=(startTime, finishtime))
# *HARD-CODES conversion from kg/m2/sec to mm/day.
tvarb *= 86400
print('Shape:', tvarb.shape)
# The following tasks need to be done only once, extracting
# metadata from first-year file:
if year == args.firstyear:
tc = tvarb.getTime().asComponentTime()
day1 = cdtime.comptime(tc[0].year, tc[0].month)
firstday = tvarb(
time=(
day1,
day1.add(
1,
cdtime.Day),
"con"))
dimensions = firstday.shape
# print ' Shape = ', dimensions
# Number of time points in the selected month for one year
N = dimensions[0]
nlats = dimensions[1]
nlons = dimensions[2]
deltaH = 24. / N
dayspermo = tvarb.shape[0] // N
print(' %d timepoints per day, %d hr intervals between timepoints' % (N, deltaH))
comptime = firstday.getTime()
modellons = tvarb.getLongitude()
modellats = tvarb.getLatitude()
# Longitude values are needed later to compute Local Solar
# Times.
lons = modellons[:]
print(' Creating temporary storage and output fields ...')
# Sorts tvarb into separate GMTs for one year
tvslice = MV2.zeros((N, dayspermo, nlats, nlons))
# Concatenates tvslice over all years
concatenation = MV2.zeros(
(N, dayspermo * nYears, nlats, nlons))
LSTs = MV2.zeros((N, nlats, nlons))
for iGMT in range(N):
hour = iGMT * deltaH + startime
print(' Computing Local Standard Times for GMT %5.2f ...' % hour)
for j in range(nlats):
for k in range(nlons):
LSTs[iGMT, j, k] = (hour + lons[k] / 15) % 24
for iGMT in range(N):
hour = iGMT * deltaH + startime
print(' Choosing timepoints with GMT %5.2f ...' % hour)
# Transient-variable slice: every Nth tpoint gets all of
# the current GMT's tpoints for current year:
tvslice[iGMT] = tvarb[iGMT:tvarb.shape[0]:N]
concatenation[iGMT, iYear *
dayspermo: (iYear +
1) *
dayspermo] = tvslice[iGMT]
iYear += 1
f.close()
# For each GMT, take mean and standard deviation over all years for
# the chosen month:
avgvalues = MV2.zeros((N, nlats, nlons))
stdvalues = MV2.zeros((N, nlats, nlons))
for iGMT in range(N):
hour = iGMT * deltaH + startime
print('Computing mean and standard deviation over all GMT %5.2f timepoints ...' % hour)
# Assumes first dimension of input ("axis#0") is time
avgvalues[iGMT] = MV2.average(concatenation[iGMT], axis=0)
stdvalues[iGMT] = genutil.statistics.std(concatenation[iGMT])
avgvalues.id = 'diurnalmean'
stdvalues.id = 'diurnalstd'
LSTs.id = 'LST'
avgvalues.units = outunits
# Standard deviation has same units as mean (not so for
# higher-moment stats).
stdvalues.units = outunits
LSTs.units = 'hr'
LSTs.longname = 'Local Solar Time'
avgvalues.setAxis(0, comptime)
avgvalues.setAxis(1, modellats)
avgvalues.setAxis(2, modellons)
stdvalues.setAxis(0, comptime)
stdvalues.setAxis(1, modellats)
stdvalues.setAxis(2, modellons)
LSTs.setAxis(0, comptime)
LSTs.setAxis(1, modellats)
LSTs.setAxis(2, modellons)
avgoutfile = ('%s_%s_%s_%s-%s_diurnal_avg.nc') % (varbname,
dataname, monthname,
str(args.firstyear), str(args.lastyear))
stdoutfile = ('%s_%s_%s_%s-%s_diurnal_std.nc') % (varbname,
dataname, monthname, str(
args.firstyear),
str(args.lastyear))
LSToutfile = ('%s_%s_LocalSolarTimes.nc' % (varbname, dataname))
if not os.path.exists(args.results_dir):
os.makedirs(args.results_dir)
f = cdms2.open(
os.path.join(
args.results_dir,
avgoutfile),
'w')
g = cdms2.open(
os.path.join(
args.results_dir,
stdoutfile),
'w')
h = cdms2.open(
os.path.join(
args.results_dir,
LSToutfile),
'w')
f.write(avgvalues)
g.write(stdvalues)
h.write(LSTs)
f.close()
g.close()
h.close()
except Exception as err:
print("Failed for model %s with erro: %s" % (dataname, err))
## asarray(data, typecode=None)
## asarray(data, typecode=None) = array(data, typecode=None, copy=0)
## Returns data if typecode if data is a MaskedArray and typecode None
## or the same.
## average(a, axis=0, weights=None, returned=0)
## average(a, axis=0, weights=None, returned=0)
## Computes average along indicated axis. Masked elements are ignored.
## Result may equal masked if average cannot be computed.
## If weights are given, result is sum(a*weights)/sum(weights), with
## all elements masked in a or in weights ignored.
## weights if given must have a's shape.
## Denominator is multiplied by 1.0 to prevent truncation for integers.
## returned governs return of second quantity, the weights.
xav = MV2.average(xones, axis=1)
xav2 = MV2.average(ud)
xav3 = MV2.average(udat)
xav4, wav4 = MV2.average(udat, weights=MV2.ones(udat.shape, numpy.float), returned=1)
## choose(indices, t)
## Shaped like indices, values t[i] where at indices[i]
## If t[j] is masked, special treatment to preserve type.
ct1 = MV2.TransientVariable([1,1,2,0,1])
ctr = MV2.choose(ct1, [numpy.ma.masked, 10,20,30,40])
if not MV2.allclose(ctr, [10, 10, 20, 100, 10]): markError('choose error 1')
ctx = MV2.TransientVariable([1,2,3,150,4])
cty = -MV2.TransientVariable([1,2,3,150,4])
ctr = MV2.choose(MV2.greater(ctx,100), (ctx, 100))
if not MV2.allclose(ctr, [1,2,3,100,4]): markError('choose error 2')
ctr = MV2.choose(MV2.greater(ctx,100), (ctx, cty))
def ensemble_average(basedir, grid = None, func = None):
models = np.unique(map(lambda x: x.split(".")[1],glob.glob(basedir+"*")))
#Deal with extremely annoying GISS physics
giss = np.where([x.find("GISS")>=0 for x in models])[0]
oldmodels = models
models = np.delete(models, giss)
for gissmo in oldmodels[giss]:
physics_versions = np.unique([x.split(".")[3][-2:] for x in glob.glob(basedir+"*"+gissmo+"*")])
for pv in physics_versions:
models = np.append(models, gissmo+" "+pv)
if grid is None: #Get the coarsest grid
the_file,grid = get_coarsest_grid(basedir)
if "CESM1-WACCM" in models:
i = np.argwhere(models == "CESM1-WACCM")
models = np.delete(models,i)
if "CanCM4" in models:
i = np.argwhere(models == "CanCM4")
models = np.delete(models,i)
mo = models[0]
print mo
ens = get_ensemble(basedir,mo)
ens0 = ens[0]
print ens0
f = cdms.open(ens0)
variable = ens0.split(".")[-4]
data = f(variable).regrid(grid,regridTool='regrid2')
cdutil.setTimeBoundsMonthly(data)
if func is not None:
data = func(data)
f.close()
time_and_space = data.shape
realizations = MV.zeros((len(ens),)+time_and_space)
realizations[0] = data
if len(ens)>1:
for i in range(len(ens))[1:]:
f = cdms.open(ens[i])
print ens[i]
data = f(variable).regrid(grid,regridTool='regrid2')
f.close()
cdutil.setTimeBoundsMonthly(data)
if func is not None:
data = func(data)
realizations[i] = data
model_average = MV.zeros((len(models),)+time_and_space)+1.e20
j= 0
model_average[j] = MV.average(realizations,axis=0)
for mo in models[1:]:
print mo
j+=1
ens = get_ensemble(basedir,mo)
realizations = MV.zeros((len(ens),)+time_and_space)
for i in range(len(ens)):
f = cdms.open(ens[i])
#print ens[i]
data = f(variable).regrid(grid,regridTool='regrid2')
f.close()
cdutil.setTimeBoundsMonthly(data)
if func is not None:
data = func(data)
print data.shape
print time_and_space
print data.shape == time_and_space
if data.shape == time_and_space:
realizations[i] = data
masked_ma = False
else:
masked_ma = True
if not masked_ma:
model_average[j] = MV.average(realizations,axis=0)
else:
print "not the right shape: "+mo
model_average[j] = MV.ones(time_and_space)+1.e20
M2 = MV.masked_where(model_average>1.e10,model_average)
M = MV.average(M2,axis=0)
M.setAxisList(data.getAxisList())
M.id = data.id
M.name = M.id
return M
# Now we can loop through the lines and look at the content
data1=[]
data2=[]
for line in lines:
# Splits the line into a list of string with seprartion
# when it finds space or tabs or return
sp=string.split(line)
# Now try to see if the first element is a number, if not skip
# we are only interested in the 2nd and third column here
try:
val1=float(sp[1]) # second column
val2=float(sp[2]) # third column
data1.append(val1)
data2.append(val2)
except:
pass # we didn't have 2 float at the begining of this line
# Now converts the 2 datasets to MV for use in other CDAT Packages
data1=MV.array(data1,id='dataset1')
data2=MV.array(data2,id='dataset2')
# Just for fun prints the average of data1
print MV.average(data1)
