def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse): """ Returns a new variable which is 'var' sub-sampled by averaging at the given samplingRate with data selected according to flagVar including the flag values specified in flagsToUse (defaults are 0 and 1). """ maskedArray=self._getMaskedArray(var, flagVar, flagsToUse) shape=var.shape if shape[1]==1: newNumArray=MV.ravel(maskedArray) newArrayMask=MV.ravel(maskedArray.mask()) newArray=MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value()) else: newArray=Numeric.zeros(shape[0], 'f') for t0 in range(shape[0]): # Set as missing if less than half are valid t1Array=maskedArray[t0] if samplingRate==1: # If half or more are good values then calculate the mean if t1Array.count()>=(shape[1]/2.): newArray[t0]=MA.average(t1Array) # otherwise set as missing value else: newArray[t0]=maskedArray.fill_value() else: raise "Averaging for non 1Hz sampling rates not yet supported!" # Now re-construct variable axes etc newTimeAxis=self._flatten2DTimeAxis(timeVar, samplingRate) newVar=self._recreateVariable(var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging") return newVar
def flatten2DTimeData(var, time_var): """ Returns a flattened 2D array variable with a recalculated time axies. """ data = MV.ravel(var) missing = var.getMissing() time_values = time_var._data time_unit = time_var.units sampling_rate = var.getAxis(1) rate = int(sampling_rate.id[3:]) newtime_values = [] for t in time_values: for i in range(rate): tvalue = t + ((1./rate)*i) newtime_values.append(tvalue) new_time_ax = cdms.createAxis(newtime_values) new_time_ax.units = time_unit new_time_ax.designateTime() new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time" atts = var.attributes newAtts = {} for att,value in atts.items(): if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value) == 1: value = value[0] if type(value) in (type(1), type(1.), type(long(1))): newAtts[att] = value elif type(value) == type("string"): newAtts[att] = value.strip() # Now create the variable newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts) return newvar
def _getMaskedArray(self, var, flagVar=None, flagValues=(0,1), missingValuesToTest=(-9999., -32767.)): """ Returns a masked array that has the values only present where the flag values are acceptable and the data itself is not missing. """ if flagVar: flagFillValue=-1 flagGTValue=max(flagValues) flagMaskWhereUnacceptable=MA.masked_greater(flagVar, flagGTValue).mask() flagMaskWhereMissing=MA.masked_equal(flagVar, flagFillValue).mask() flagMask=flagMaskWhereUnacceptable+flagMaskWhereMissing for fv in missingValuesToTest: if fv in MV.ravel(var): print "Setting missing value for '%s' as: %s" % (var.id, fv) varFillValue=fv else: varFillValue=missingValuesToTest[0] if flagVar: varMask=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask() fullmask=MA.bitwise_or(varMask, flagMask) maskedArray=MA.masked_array(var, mask=fullmask, fill_value=varFillValue) else: maskedArray=MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue) #maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue) #maskedArray.missing_value=varFillValue return maskedArray
def _getMaskedArray(self, var, flagVar=None, flagValues=(0, 1), missingValuesToTest=(-9999.0, -32767.0)): """ Returns a masked array that has the values only present where the flag values are acceptable and the data itself is not missing. """ if flagVar: flagFillValue = -1 flagGTValue = max(flagValues) flagMaskWhereUnacceptable = MA.masked_greater(flagVar, flagGTValue).mask() flagMaskWhereMissing = MA.masked_equal(flagVar, flagFillValue).mask() flagMask = flagMaskWhereUnacceptable + flagMaskWhereMissing for fv in missingValuesToTest: if fv in MV.ravel(var): print "Setting missing value for '%s' as: %s" % (var.id, fv) varFillValue = fv else: varFillValue = missingValuesToTest[0] if flagVar: varMask = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue).mask() fullmask = MA.bitwise_or(varMask, flagMask) maskedArray = MA.masked_array(var, mask=fullmask, fill_value=varFillValue) else: maskedArray = MA.masked_array(var, mask=MA.equal(var, varFillValue), fill_value=varFillValue) # maskedArray=cdms.createVariable(maskedArray, id=var.id, fill_value=varFillValue) # maskedArray.missing_value=varFillValue return maskedArray
def _subSampleByAveraging(self, var, timeVar, flagVar, samplingRate, flagsToUse): """ Returns a new variable which is 'var' sub-sampled by averaging at the given samplingRate with data selected according to flagVar including the flag values specified in flagsToUse (defaults are 0 and 1). """ maskedArray = self._getMaskedArray(var, flagVar, flagsToUse) shape = var.shape if shape[1] == 1: newNumArray = MV.ravel(maskedArray) newArrayMask = MV.ravel(maskedArray.mask()) newArray = MA.masked_array(newNumArray, mask=newArrayMask, fill_value=maskedArray.fill_value()) else: newArray = Numeric.zeros(shape[0], "f") for t0 in range(shape[0]): # Set as missing if less than half are valid t1Array = maskedArray[t0] if samplingRate == 1: # If half or more are good values then calculate the mean if t1Array.count() >= (shape[1] / 2.0): newArray[t0] = MA.average(t1Array) # otherwise set as missing value else: newArray[t0] = maskedArray.fill_value() else: raise "Averaging for non 1Hz sampling rates not yet supported!" # Now re-construct variable axes etc newTimeAxis = self._flatten2DTimeAxis(timeVar, samplingRate) newVar = self._recreateVariable( var, newArray, newTimeAxis, flagVar, max(flagsToUse), missingValue=maskedArray.fill_value(), sampleBy="averaging", ) return newVar
def flatten2DTimeData(var, time_var): """ Returns a flattened 2D array variable with a recalculated time axies. """ data = MV.ravel(var) missing = var.getMissing() time_values = time_var._data time_unit = time_var.units sampling_rate = var.getAxis(1) rate = int(sampling_rate.id[3:]) newtime_values = [] for t in time_values: for i in range(rate): tvalue = t + ((1. / rate) * i) newtime_values.append(tvalue) new_time_ax = cdms.createAxis(newtime_values) new_time_ax.units = time_unit new_time_ax.designateTime() new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time" atts = var.attributes newAtts = {} for att, value in atts.items(): if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array( [1.]))) and len(value) == 1: value = value[0] if type(value) in (type(1), type(1.), type(long(1))): newAtts[att] = value elif type(value) == type("string"): newAtts[att] = value.strip() # Now create the variable newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts) return newvar