def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse):
"""
Returns a new variable which is 'var' sub-sampled by averaging
at the given samplingRate with data selected according to flagVar
including the flag values specified in flagsToUse (defaults are 0 and 1).
"""
maskedArray=self._getMaskedArray(var, flagVar, flagsToUse)
shape=var.shape
if shape[1]==1:
newNumArray=MV.ravel(maskedArray)
newArrayMask=MV.ravel(maskedArray.mask())
newArray=MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value())
else:
newArray=Numeric.zeros(shape[0], 'f')
for t0 in range(shape[0]):
# Set as missing if less than half are valid
t1Array=maskedArray[t0]
if samplingRate==1:
# If half or more are good values then calculate the mean
if t1Array.count()>=(shape[1]/2.):
newArray[t0]=MA.average(t1Array)
# otherwise set as missing value
else:
newArray[t0]=maskedArray.fill_value()
else:
raise "Averaging for non 1Hz sampling rates not yet supported!"
# Now re-construct variable axes etc
newTimeAxis=self._flatten2DTimeAxis(timeVar, samplingRate)
newVar=self._recreateVariable(var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging")
return newVar
def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse):
"""
Returns a new variable which is 'var' sub-sampled by averaging
at the given samplingRate with data selected according to flagVar
including the flag values specified in flagsToUse (defaults are 0 and 1).
"""
maskedArray = self._getMaskedArray(var, flagVar, flagsToUse)
shape = var.shape
if shape[1] == 1:
newNumArray = MV.ravel(maskedArray)
newArrayMask = MV.ravel(maskedArray.mask())
newArray = MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value())
else:
newArray = Numeric.zeros(shape[0], "f")
for t0 in range(shape[0]):
# Set as missing if less than half are valid
t1Array = maskedArray[t0]
if samplingRate == 1:
# If half or more are good values then calculate the mean
if t1Array.count() >= (shape[1] / 2.0):
newArray[t0] = MA.average(t1Array)
# otherwise set as missing value
else:
newArray[t0] = maskedArray.fill_value()
else:
raise "Averaging for non 1Hz sampling rates not yet supported!"
# Now re-construct variable axes etc
newTimeAxis = self._flatten2DTimeAxis(timeVar, samplingRate)
newVar = self._recreateVariable(
var,
newArray,
newTimeAxis,
flagVar,
max(flagsToUse),
missingValue=maskedArray.fill_value(),
sampleBy="averaging",
)
return newVar
def processRainfall(file, outdir, var, north, west, south, east):
"Subsets, averages, writes to binary files."
f=cdms.open(file)
v=f(var, lat=(south, north), lon=(west, east))
timevalues=v.getTime()[:]
t0=timevalues[0]
# I need to test if step 0 always has only missing values
# remove -50 values???
v=MA.masked_less(v,0)
# create average of all ensemble members
av=MA.average(v, axis=1)
# get stuff for name
datetime=os.path.split(file)[-1].split(".")[1]
outpaths=[]
# now step through time dimension (0)
count=0
for dslice in av:
ts=timevalues[count]-t0
outfile="rainfall.%s.%dh.dat" % (datetime, ts)
outpath=os.path.join(outdir, outfile)
count=count+1
numarray=Numeric.array(dslice._data)
sh=numarray.shape
length=sh[0]*sh[1]
flatarray=Numeric.resize(numarray, [length])
output=open(outpath, "wb")
arr=array.array('f', flatarray)
arr.tofile(output)
output.close()
print "Written:", outpath
outpaths.append(outpath)
return outpaths
# How to use numpy with 'None' value in Python? import MA a = MA.array([1, 2, None], mask = [0, 0, 1]) print "average =", MA.average(a)
hlat = ice1.variables["hlat"] # hlat[49]
hlon = ice1.variables["hlon"] # hlon[100]
dimf = fice.shape # Define an array to hold long-term monthly means.
ntime = fice.shape[0]
nhlat = fice.shape[1]
nhlon = fice.shape[2]
nmo = 0
month = nmo+1
icemon = MA.zeros((nhlat,nhlon),MA.Float0)
for i in xrange(fice_masked.shape[0]):
for j in xrange(fice_masked.shape[1]):
icemon[i,j] = MA.average(fice_masked[i,j,0:ntime:12])
#
# Fill the places where icemon is zero with the fill value.
#
icemon = MA.masked_values(icemon,0.,rtol=0.,atol=1.e-15)
icemon = MA.filled(icemon,value=fill_value)
# Calculate the January (nmo=0) average.
nsub = 16 # Subscript location of northernmost hlat to be plotted.
cmap = Numeric.array([ \
[1.00,1.00,1.00], [0.00,0.00,0.00], [1.00,1.00,0.50], \
[0.00,0.00,0.50], [0.50,1.00,1.00], [0.50,0.00,0.00], \
from soomarray import ArrayDict
ad = ArrayDict('blah.dat', 'r+')
a = Numeric.array([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], Numeric.Int)
m = Numeric.array([0, 0, 0, 0, 0, 1, 0, 0, 1, 0], Numeric.Int)
ad['matest1'] = MA.array(a, mask=m)
del ad
ad = ArrayDict('blah.dat')
matest = ad['matest1']
print "matest: ", matest
print "sum of matest: ", MA.sum(matest)
print "length of matest: ", len(matest)
print "count of matest: ", MA.count(matest)
print "average of matest: ", MA.average(matest)
print "minimum of matest: ", MA.minimum(matest)
print "maximum of matest: ", MA.maximum(matest)
del ad
ad = ArrayDict('blah.dat', 'w')
a = Numeric.array(xrange(1000), Numeric.Int)
m = Numeric.array(Numeric.repeat(Numeric.array([0, 1], Numeric.Int), 500),
Numeric.Int)
ad['matest2'] = MA.array(a, mask=m)
m = Numeric.array(Numeric.repeat(Numeric.array([1, 0], Numeric.Int), 500),
Numeric.Int)
ad['matest3'] = MA.array(a, mask=m)
def get_ascii_xy(self,rq,variable=None, mime='html'):
html = 'trying <br/>'
if not self.check_open():
return (1, 'failed to open file',' ','could not open %s<br/>' % self.ddd.this_file)
pmess = ' xxx '
try:
nc = self.nc
vk = nc.variables.keys()
html += '%s <br/>' % str( vk )
nsn = 0
for vv in vk:
if len( nc.variables[vv].shape ) == 3:
html += '%s <br/>' % str(nc.variables[vv].attributes.keys())
standard_name = nc.variables[vv].attributes.get('standard_name', nc.variables[vv].attributes['name'] )
v = vv
nsn += 1
if nsn != 1:
if nsn > 1:
return (0,' ', ' ','more than one named rank 3 variable in file')
else:
return (0,' ', ' ','no named rank 3 variable in file')
except:
return (1, 'failed to identify variable',' ',html)
self.variable_name = v
if nc.variables[v].attributes.has_key('standard_name'):
self.standard_name = standard_name
else:
self.standard_name = None
if mime == 'csv':
self.make_csv_header()
try:
html += '%s<br/>' % standard_name
var = nc.variables[v]
data = nc.variables[v].getValue()
Axes = nc.variables[v].getAxisList()
fmt = fmts.get( v, '%s' )
tAx = nc.getAxis( 'time' )
yAx = nc.getAxis( 'latitude' )
xAx = nc.getAxis( 'longitude' )
yAxes = yAx.getValue()
xAxes = xAx.getValue()
if rq.fields.has_key( 'forwardbutton' ):
lat0 = iget( rq, 'startY', 0 )
lat9 = iget( rq, 'endY', min(len(yAxes)-1,5) )
lon0 = iget( rq, 'startX', 0 )
lon9 = iget( rq, 'endX', min(len(xAxes)-1,5) )
xtarg0 = xAxes[lon0]
xtarg9 = xAxes[lon9]
ytarg0 = yAxes[lat0]
ytarg9 = yAxes[lat9]
else:
ytarg0, lat0 = ccget( rq, 'YY0', '0.00', yAxes )
ytarg9, lat9 = ccget( rq, 'YY9', '30.00', yAxes )
xtarg0, lon0 = ccget( rq, 'XX0', '0.00', xAxes )
xtarg9, lon9 = ccget( rq, 'XX9', '30.00', xAxes )
##if ix > len( xAxes ) -1:
##ix = len( xAxes ) -1
##if iy > len( yAxes ) -1:
##iy = len( yAxes ) -1
html += 'zz'
startYear, startMonth, startDay, endYear = utils.cf_time_info( nc, tAx, bounds='clim_bounds' )
trange = '%s-%s' % (startYear, endYear )
self.time_info = (startYear, startMonth, startDay, endYear)
kt=0
km=0
plot_mean = False
try:
if rq.fields.has_key( 'monthSeason'):
msv = '??'
try:
msv = rq.fields['monthSeason'].value
if msv in ['djf','mam','jja','son','annual']:
plot_mean = True
else:
plot_mean = False
km = int(msv) - 1
except:
msv = '???'
else:
msv = 'unset'
except:
msv = 'could not set'
if msv == 'unset':
cmsv = 'Jan'
elif plot_mean:
cmsv = msv
else:
cmsv = mnths[km]
html += 'msv = %s<br/>' % str(msv)
sel = { True:' selected="selected"', False:'' }
sel2 = { True:' disabled="disabled"', False:'' }
if mime == 'csv':
self.csv_header_append( 'Attribute','g','Time period in year', cmsv )
self.csv_header_append( 'Attribute','g','Start year', startYear )
self.csv_header_append( 'Attribute','g','End year', endYear )
il = range(len(yAxes))
if yAxes[-1] > yAxes[0]:
il.reverse()
displayModes = ''
dm2 = '<center>Select display mode:<br/>'
if mime == 'html':
try:
dm = ['xy','ts']
dmn = ['Latitude-Longitude grid','Time series at a point']
for k in range(len(dm)):
displayModes += \
'<option value="%s"%s>%s</option>\n' % (k+1,sel[k==0],dmn[k])
dm2 += '<input type="submit"%s name="*%s" value="%s"/><br/>\n' % (sel2[k==0],dm[k],dmn[k])
except:
displayModes = 'zzzzzzzzzzzzzzzzzzzz'
dm2 += "</center>"
time_options = ' '
for k in range(12):
time_options += '<option value="%s"%s>%s</option>\n' % (k+1,sel[km==k],mnths[k])
for k in range(len(seas)):
time_options += '<option value="%s"%s>%s</option>\n' % (seas[k],sel[msv==seas[k]],seas[k])
start_y_options =' '
end_y_options =' '
end_y_label = 'Upper Latitude'
start_y_label = 'Lower Latitude'
for k in il:
start_y_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==lat0],yAxes[k])
end_y_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==lat9],yAxes[k])
end_x_label = 'Upper Longitude'
start_x_label = 'Lower Longitude'
start_x_options =' '
end_x_options =' '
for k in range(len(xAxes)):
start_x_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==lon0],xAxes[k])
end_x_options += \
'<option value="%s"%s>%6.2f</option>\n' % (k+1,sel[k==lon9],xAxes[k])
yname = axname( yAx, 'Y' )
xname = axname( xAx, 'X' )
if mime == 'csv':
self.csv_header_append( 'Comment','First column',yname, ' ' )
self.csv_header_append( 'Comment','First row',xname, ' ' )
html += 'axes: %s, %s<br/>' % (xname,yname)
s = data.shape
sl = [0,0,0]
islt = Axes.index( tAx )
isly = Axes.index( yAx )
islx = Axes.index( xAx )
if lat0 > lat9:
yslice = slice( lat9, lat0+1 )
else:
yslice = slice( lat0, lat9+1 )
sl[islt] = km
sl[isly] = yslice
sl[islx] = slice( lon0, lon9+1 )
if standard_name in ['precipitation_flux_anomaly','precipitation_flux']:
units = 'mm/day'
factor = 3600*24
elif standard_name in ['precipitation_amount']:
units = 'mm/month'
factor = 1
else:
factor = 1
units = var.attributes['units']
if standard_name == 'air_temperature' and units == 'hPa':
units = 'Celsius'
if variable in utils.cru_vdict.keys():
display_name = utils.cru_vdict[variable]
elif variable in utils.tar_vdict.keys():
display_name = utils.tar_vdict[variable]
else:
display_name = standard_name
title = '%s [%s], %s' % (display_name,units,cmsv)
html += '<br/>xxx'
html += '<br/>' + str(sl)
html += '<br/>' + str(data[sl].shape)
html += '<br/>' + xname
html += '<br/>' + yname
html += '<br/>' + str(xAxes[lon0:lon9+1])
html += '<br/>' + str(plot_mean)
if not plot_mean:
data_display = slice2table( title, xname, xAxes[lon0:lon9+1], yname, yAxes[yslice], data[sl], isly > islx, fmt, factor, mime=mime )
else:
html += '<br/>' + str(msv)
t = { 'annual':(0,12), 'mam':(2,5), 'jja':(5,8),'son':(8,11),'djf':[(11,12),(0,2)] }[msv]
html += '<br/> time slice: ' + str(t)
pmess += '<br/> %s <br/>' % str(t)
pmess += '%s<br/>\n' % str( type )
pmess += '%s<br/>\n' % str( t[0] )
try:
import Numeric
import MA as thisMa
except:
import numpy
import numpy.ma as thisMa
if type( t[0] ) == type(0):
sl[islt] = slice( t[0],t[1] )
html += '<br/>' + str(sl)
html += '<br/> data shape:: ' + str(data.shape)
pmess += 'zzzz<br/>\n'
vv = thisMa.masked_outside( data[sl].tolist(), -990., 900. )
v = thisMa.average( vv, axis=islt )
v = thisMa.masked_values( v, -999. )
else:
sl1 = sl[:]
sl[islt] = slice(t[0][0],t[0][1])
sl1[islt] = slice(t[1][0],t[1][1])
pmess += 'qqqq<br/>\n'
vv = thisMa.masked_outside( data[sl].tolist(), -990., 900. )
vv1 = thisMa.masked_outside( data[sl1].tolist(), -990., 900. )
pmess += 'qqqq<br/>\n'
v = (thisMa.sum( vv, axis=islt ) + thisMa.sum( vv1, axis=islt ) )/3.
v = thisMa.masked_values( v, -999. )
pmess += 'qqqq<br/>\n'
html += '<br/> calling slice2table'
data_display = slice2table( title, xname, xAxes[lon0:lon9+1], yname, yAxes[yslice], v, isly > islx, fmt, factor, mime=mime )
pmess += 'qqqq<br/>\n'
html += '<br/> back from slice2table'
if mime == 'html':
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('YY0',ytarg0)
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('YY9',ytarg9)
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('XX0',xtarg0)
data_display += '<input type="hidden" name="%s" value="%7.3f"/>\n' % ('XX9',xtarg9)
html += '<br/>display generated'
if mime == 'html':
try:
x = 0
import os
ii = open( self.tmpl_xyt1, 'r' )
x = 3
html_pat = ii.readlines()
x = 1
html_pat = string.join( html_pat )
x = 2
table_html = html_pat % locals()
except:
if x == 2:
data_display += '<br/> could not generate table'
elif x == 1:
data_display += '<br/> could not join template'
elif x == 3:
data_display += '<br/> could not read template'
else:
data_display += '<br/> could _open template'
table_html = ' '
else:
table_html = ' '
rv = 0
table_html += dm2
pmess += str( data.shape )
pmess += '<br/> %s ' % islt
return (rv,html,table_html, data_display)
except:
return (1, 'failed',' ',html + pmess)
