def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse):
"""
Returns a new variable which is 'var' sub-sampled by averaging
at the given samplingRate with data selected according to flagVar
including the flag values specified in flagsToUse (defaults are 0 and 1).
"""
maskedArray=self._getMaskedArray(var, flagVar, flagsToUse)
shape=var.shape
if shape[1]==1:
newNumArray=MV.ravel(maskedArray)
newArrayMask=MV.ravel(maskedArray.mask())
newArray=MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value())
else:
newArray=Numeric.zeros(shape[0], 'f')
for t0 in range(shape[0]):
# Set as missing if less than half are valid
t1Array=maskedArray[t0]
if samplingRate==1:
# If half or more are good values then calculate the mean
if t1Array.count()>=(shape[1]/2.):
newArray[t0]=MA.average(t1Array)
# otherwise set as missing value
else:
newArray[t0]=maskedArray.fill_value()
else:
raise "Averaging for non 1Hz sampling rates not yet supported!"
# Now re-construct variable axes etc
newTimeAxis=self._flatten2DTimeAxis(timeVar, samplingRate)
newVar=self._recreateVariable(var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging")
return newVar
def flatten2DTimeData(var, time_var):
"""
Returns a flattened 2D array variable with a recalculated time axies.
"""
data = MV.ravel(var)
missing = var.getMissing()
time_values = time_var._data
time_unit = time_var.units
sampling_rate = var.getAxis(1)
rate = int(sampling_rate.id[3:])
newtime_values = []
for t in time_values:
for i in range(rate):
tvalue = t + ((1./rate)*i)
newtime_values.append(tvalue)
new_time_ax = cdms.createAxis(newtime_values)
new_time_ax.units = time_unit
new_time_ax.designateTime()
new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time"
atts = var.attributes
newAtts = {}
for att,value in atts.items():
if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value) == 1:
value = value[0]
if type(value) in (type(1), type(1.), type(long(1))):
newAtts[att] = value
elif type(value) == type("string"):
newAtts[att] = value.strip()
# Now create the variable
newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts)
return newvar
def _getMaskedArray(self, var, flagVar=None, flagValues=(0,1), missingValuesToTest=(-9999., -32767.)):
"""
Returns a masked array that has the values only present where the flag
values are acceptable and the data itself is not missing.
"""
if flagVar:
flagFillValue=-1
flagGTValue=max(flagValues)
flagMaskWhereUnacceptable=MA.masked_greater(flagVar, flagGTValue).mask()
flagMaskWhereMissing=MA.masked_equal(flagVar, flagFillValue).mask()
flagMask=flagMaskWhereUnacceptable+flagMaskWhereMissing
for fv in missingValuesToTest:
if fv in MV.ravel(var):
print "Setting missing value for '%s' as: %s" % (var.id, fv)
varFillValue=fv
else:
varFillValue=missingValuesToTest[0]
if flagVar:
varMask=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask()
fullmask=MA.bitwise_or(varMask, flagMask)
maskedArray=MA.masked_array(var, mask=fullmask, fill_value=varFillValue)
else:
maskedArray=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue)
#maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue)
#maskedArray.missing_value=varFillValue
return maskedArray
def _getMaskedArray(self, var, flagVar=None, flagValues=(0, 1), missingValuesToTest=(-9999.0, -32767.0)):
"""
Returns a masked array that has the values only present where the flag
values are acceptable and the data itself is not missing.
"""
if flagVar:
flagFillValue = -1
flagGTValue = max(flagValues)
flagMaskWhereUnacceptable = MA.masked_greater(flagVar, flagGTValue).mask()
flagMaskWhereMissing = MA.masked_equal(flagVar, flagFillValue).mask()
flagMask = flagMaskWhereUnacceptable + flagMaskWhereMissing
for fv in missingValuesToTest:
if fv in MV.ravel(var):
print "Setting missing value for '%s' as: %s" % (var.id, fv)
varFillValue = fv
else:
varFillValue = missingValuesToTest[0]
if flagVar:
varMask = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask()
fullmask = MA.bitwise_or(varMask, flagMask)
maskedArray = MA.masked_array(var, mask=fullmask, fill_value=varFillValue)
else:
maskedArray = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue)
# maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue)
# maskedArray.missing_value=varFillValue
return maskedArray
def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse):
"""
Returns a new variable which is 'var' sub-sampled by averaging
at the given samplingRate with data selected according to flagVar
including the flag values specified in flagsToUse (defaults are 0 and 1).
"""
maskedArray = self._getMaskedArray(var, flagVar, flagsToUse)
shape = var.shape
if shape[1] == 1:
newNumArray = MV.ravel(maskedArray)
newArrayMask = MV.ravel(maskedArray.mask())
newArray = MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value())
else:
newArray = Numeric.zeros(shape[0], "f")
for t0 in range(shape[0]):
# Set as missing if less than half are valid
t1Array = maskedArray[t0]
if samplingRate == 1:
# If half or more are good values then calculate the mean
if t1Array.count() >= (shape[1] / 2.0):
newArray[t0] = MA.average(t1Array)
# otherwise set as missing value
else:
newArray[t0] = maskedArray.fill_value()
else:
raise "Averaging for non 1Hz sampling rates not yet supported!"
# Now re-construct variable axes etc
newTimeAxis = self._flatten2DTimeAxis(timeVar, samplingRate)
newVar = self._recreateVariable(
var,
newArray,
newTimeAxis,
flagVar,
max(flagsToUse),
missingValue=maskedArray.fill_value(),
sampleBy="averaging",
)
return newVar
def flatten2DTimeData(var, time_var):
"""
Returns a flattened 2D array variable with a recalculated time axies.
"""
data = MV.ravel(var)
missing = var.getMissing()
time_values = time_var._data
time_unit = time_var.units
sampling_rate = var.getAxis(1)
rate = int(sampling_rate.id[3:])
newtime_values = []
for t in time_values:
for i in range(rate):
tvalue = t + ((1. / rate) * i)
newtime_values.append(tvalue)
new_time_ax = cdms.createAxis(newtime_values)
new_time_ax.units = time_unit
new_time_ax.designateTime()
new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time"
atts = var.attributes
newAtts = {}
for att, value in atts.items():
if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array(
[1.]))) and len(value) == 1:
value = value[0]
if type(value) in (type(1), type(1.), type(long(1))):
newAtts[att] = value
elif type(value) == type("string"):
newAtts[att] = value.strip()
# Now create the variable
newvar = cdms.createVariable(data,
id=var.id,
fill_value=missing,
axes=[new_time_ax],
attributes=newAtts)
return newvar