def getSamplesAtTimes(grid,
year=None,
season=None,
mon=None,
day=None,
hour=None,
min=None,
sec=None,
ms=None):
""" Samples a grid at specified time periods, multiple arguments can be used in complex sampling
eg.., using hour = 5 would return all samples corresponding to 5 am, further specifing year = 2008
would give samples at 5am in year 2008
"""
from visad import RealType
from visad import Gridded1DDoubleSet
from visad import FieldImpl
from visad import FunctionType
from visad import VisADException
if (str(mon) != "None" and str(season) != "None"):
raise VisADException("One of Month or Season can be used, not both")
timeSet = GridUtil.getTimeSet(grid)
indices = getSampleTimeIndices(grid, year, season, mon, day, hour, min,
sec, ms)
timevals = timeSet.getSamples()[0]
subsetTimeValues = [timevals[i] for i in indices]
newTimes = Gridded1DDoubleSet(RealType.Time, [subsetTimeValues],
len(subsetTimeValues), None,
timeSet.getSetUnits(), None)
subsetgrid = FieldImpl(
FunctionType(RealType.Time,
grid.getSample(0).getType()), newTimes)
for i in range(len(subsetTimeValues)):
subsetgrid.setSample(i, grid[indices[i]])
return subsetgrid
def makeTimeComposite(variable, avgvariable, minvalue, maxvalue):
""" Make a time composite of grid supplied(variable) between min max ranges
of a 1d avg variable supplied.
"""
from visad import FunctionType
from visad import FieldImpl
from visad import RealType
from visad import Gridded1DDoubleSet
from visad import VisADException
from ucar.unidata.util.Misc import getAverage
timeSet = GridUtil.getTimeSet(avgvariable)
newTimeIndexList = java.util.ArrayList()
newTimeValues = []
for i in range(timeSet.getLength()):
avg = getAverage(avgvariable.getSample(i).getFloats()[0])
if minvalue < avg <= maxvalue:
newTimeIndexList.add(Integer(i))
newTimeValues.append(timeSet[i].getValue())
print(len(newTimeIndexList))
if (len(newTimeIndexList) < 1):
raise VisADException("No Matches found to make a time composite")
newTimes = Gridded1DDoubleSet(RealType.Time, [newTimeValues],
len(newTimeValues), None,
timeSet.getSetUnits(), None)
compvariable = FieldImpl(
FunctionType(RealType.Time,
variable.getSample(0).getType()), newTimes)
for i in range(len(newTimeValues)):
compvariable.setSample(i, variable[newTimeIndexList[i]])
return compvariable
def anomalyFromTimeMeans(grid, meanType="None"):
""" Returns deviation from time means, timemean can be monthly, daily etc..
eg.., meanType="day" will return deviation of each step from its
corresponding daily mean.
"""
from visad import Real
from visad import Gridded1DDoubleSet
from visad import FieldImpl
from visad import FunctionType
from visad import RealType
from visad.Data import NEAREST_NEIGHBOR
from visad.Data import NO_ERRORS
from visad import VisADException
timeMean = createTimeMeans(grid, meanType)
grid.subtract(timeMean)
if (str(meanType).lower() in ("mon", "month", "months", "montly")):
searchFormat = "MM"
elif (str(meanType).lower() in ("day", "d", "days", "daily")):
searchFormat = "dd"
elif (str(meanType).lower() in ("hr", "hour", "hours", "hourly")):
searchFormat = "hh"
elif (str(meanType).lower()
in ("m", "min", "minute", "minutes", "minutely")):
searchFormat = "mm"
elif (str(meanType).lower() in ("s", "sec", "second", "seconds")):
searchFormat = "ss"
else:
raise VisADException(
"Unrecognized time mean type, use yearly or monthly etc")
return grid.subtract(timeMean, NEAREST_NEIGHBOR, NO_ERRORS)
def getSampleTimesInFormat(grid,
timeformat,
timezone="UTC",
outformat="string"):
""" A Helper function to return times of a grid in specified format as a list.
"""
from ucar.visad.data import CalendarDateTime
from visad import DateTime
from ucar.visad.UtcDate import formatUtcDate
from visad import VisADException
from java.util import TimeZone
dateTimes = CalendarDateTime.timeSetToArray(GridUtil.getTimeSet(grid))
TIMEZONE = TimeZone.getTimeZone(timezone)
temp = []
for i in range(grid.getDomainSet().getLength()):
if (str(outformat).lower() in ("string", "str")):
temp.append(str(dateTimes[i].formattedString(timeformat,
TIMEZONE)))
elif (str(outformat).lower() in ("float", "flt")):
temp.append(
float(dateTimes[i].formattedString(timeformat, TIMEZONE)))
elif (str(outformat).lower() in ("int", "integer")):
temp.append(
Integer(dateTimes[i].formattedString(timeformat, TIMEZONE)))
else:
raise VisADException("Unrecognized output format")
return temp
def calcMonAnom(monthly, ltm, normalize=0):
""" Calculate the monthly anomaly from a long term mean.
The number of timesteps in ltm must be 12
"""
from visad import VisADException
monAnom = monthly.clone()
months = len(ltm)
if (not months == 12):
raise VisADException("Number of months in ltm must be a 12")
years = int(len(monthly) / months) + 1
startMonth = getStartMonth(GridUtil.getTimeSet(monthly)) - 1
#print "Start month = " , startMonth
index = 0
for year in range(years):
for month in range(12):
if index > len(monthly) - 1:
break
thisMonth = (startMonth + month) % 12
#print thisMonth
diff = sub(monthly[index], ltm[thisMonth])
if normalize != 0:
diff = sub(diff, xav(diff))
diff = GridUtil.setParamType(diff,
GridUtil.getParamType(monAnom))
monAnom[index] = diff
index = index + 1
return monAnom
def cdo2(variable1, variable2, user_cdo_options):
""" Does a cdo operation on 2 grids supplied and returns a grid.
The function outsources the two grids to CDOs (code.zmaw.de/cdo) user option decides what
operation can be done.
evalues in cdo as cdo user_cdo_options variable1_written_to_temp_file.nc variable2_written.nc output_temp_file.nc
"""
import java
from java.util import Random
import sys
import commands
from visad import VisADException
rand = Random()
file_prefix = rand.nextInt(9999)
file_inp1 = str(file_prefix) + "_inp1.nc"
file_inp2 = str(file_prefix) + "_inp2.nc"
exportGridToNetcdf(variable1, file_inp1)
exportGridToNetcdf(variable2, file_inp2)
timeunit1 = GridUtil.getTimeSet(variable1).getSetUnits()[0]
timeunit2 = GridUtil.getTimeSet(variable2).getSetUnits()[0]
if (str(timeunit1).split()[0] == "s"):
newtimeunit1 = str(timeunit1).replace("s", "seconds", 1)
nco_status = commands.getstatusoutput("ncatted -a units,time,m,c," +
"\"" + newtimeunit1 + "\"" +
" " + file_inp1)
if (nco_status[0]):
raise VisADException(nco_status[1])
if (str(timeunit2).split()[0] == "s"):
newtimeunit2 = str(timeunit2).replace("s", "seconds", 1)
nco_status = commands.getstatusoutput("ncatted -a units,time,m,c," +
"\"" + newtimeunit2 + "\"" +
" " + file_inp2)
if (nco_status[0]):
raise VisADException(nco_status[1])
file_out = str(file_prefix) + "_out.nc"
cdo_status = commands.getstatusoutput("cdo " + user_cdo_options + " " +
file_inp1 + " " + file_inp2 + " " +
file_out)
if (cdo_status[0]):
raise VisADException(cdo_status[1])
dstemp = makeDataSource(file_out)
varout = getData(dstemp.getName())
return varout
def cdoSubGrid(variable, user_nlon, user_nlat):
""" Computes a subgrid difference between a fine resolution and
coarsened resolution grid (from regriding the
original grid to user_nlon and user_nlat) using CDOs.Works
best for regular grids.
Note: Also needs ncatted from NCO's also to be on your path.
"""
import java
from java.util import Random
import sys
import commands
import os
try:
os.remove(idv.getObjectStore().getJythonCacheDir() +
"/Lib/threading.py")
except:
pass
from visad import VisADException
rand = Random()
file_prefix = rand.nextInt(9999)
file_inp = str(file_prefix) + "_inp.nc"
exportGridToNetcdf(variable, file_inp)
timeunit = GridUtil.getTimeSet(variable).getSetUnits()[0]
if (str(timeunit).split()[0] == "s"):
newtimeunit = str(timeunit).replace("s", "seconds", 1)
nco_status = commands.getstatusoutput("ncatted -a units,time,m,c," +
"\"" + newtimeunit + "\"" + " " +
file_inp)
if (nco_status[0]):
raise VisADException(nco_status[1])
file_out = str(file_prefix) + "_out.nc"
cdo_command = "cdo sub " + file_inp + " -remapnn," + file_inp + " -remapcon,r" + user_nlon + "x" + user_nlat + " " + file_inp + " " + file_out
cdo_status = commands.getstatusoutput(cdo_command)
if (cdo_status[0]):
raise VisADException(cdo_status[1])
dstemp = makeDataSource(file_out)
varout = getData(dstemp.getName())
return varout
def ddz(grid):
""" Computes a vertical coordinate derivative of grid specifed.
"""
from visad import VisADException
#doesnt work well to raise exception for case when one level is present,
#in that case does derivative for longitude
if (GridUtil.is3D(GridUtil.getSpatialDomain(grid))):
leveldim = GridUtil.getSpatialDomain(grid).getManifoldDimension()
else:
raise VisADException("Not a 3D Spatial Grid")
return GridMath.partial(grid, (leveldim - 1))
def getLevels(grid):
""" A helper function to get levels values inside a grid as a list.
"""
from visad import VisADException
if GridUtil.is3D(GridUtil.getSpatialDomain(grid)):
leveldim = GridUtil.getSpatialDomain(grid).getManifoldDimension()
levels = remove_duplicates(
GridUtil.getSpatialDomain(grid).getSamples()[leveldim - 1])
else:
raise VisADException("No Vertical Levels Found")
return levels
def getTimeDict(grid):
""" A helper function to return timestamps of grid as dictionary of years, months etc.
"""
from ucar.visad.data import CalendarDateTime
from visad import DateTime
from ucar.visad.UtcDate import formatUtcDate
if (GridUtil.isTimeSequence(grid) == 1):
dateTimes = CalendarDateTime.timeSetToArray(grid.getDomainSet())
YYYY = []
MM = []
dd = []
DD = []
mm = []
hh = []
ss = []
YYYYMMdd = []
HHmmss = []
for i in range(ds.getDomainSet().getLength()):
print formatUtcDate(dateTimes[i], "DD", DateTime.DEFAULT_TIMEZONE)
YYYY.append(
str(dateTimes[i].formattedString("yyyy",
DateTime.DEFAULT_TIMEZONE)))
MM.append(
str(dateTimes[i].formattedString("MM",
DateTime.DEFAULT_TIMEZONE)))
dd.append(
str(dateTimes[i].formattedString("dd",
DateTime.DEFAULT_TIMEZONE)))
DD.append(
str(dateTimes[i].formattedString("DD",
DateTime.DEFAULT_TIMEZONE)))
hh.append(
str(dateTimes[i].formattedString("HH",
DateTime.DEFAULT_TIMEZONE)))
mm.append(
str(dateTimes[i].formattedString("mm",
DateTime.DEFAULT_TIMEZONE)))
ss.append(
str(dateTimes[i].formattedString("ss",
DateTime.DEFAULT_TIMEZONE)))
YYYYMMdd.append(
str(dateTimes[i].formattedString("YYYYMMdd",
DateTime.DEFAULT_TIMEZONE)))
HHmmss.append(
str(dateTimes[i].formattedString("HHmmss",
DateTime.DEFAULT_TIMEZONE)))
timeDict = dict([('YYYYMMdd', YYYYMMdd), ('HHmmss', HHmmss),
('YYYY', YYYY), ('MM', MM), ('dd', dd), ('DD', DD),
('mm', mm), ('ss', ss)])
else:
raise VisADException("This grid is not a time sequence")
return timeDict
def correlation(xvar, yvar):
""" Computes time correlation at each grid point in xvar with corresponding grid
in yvar.
"""
if (GridUtil.isTimeSequence(xvar) and GridUtil.isTimeSequence(yvar)
and GridUtil.getTimeSet(xvar).getLength()
== GridUtil.getTimeSet(yvar).getLength()):
xavg = averageOverTime(xvar, makeTimes=0)
yavg = averageOverTime(yvar, makeTimes=0)
xdev = xvar - xavg
ydev = yvar - yavg
xydevsumbyn = sumOverTime(
xdev.multiply(ydev)) / (GridUtil.getTimeSet(xdev).getLength() - 1)
xstddev = sumOverTime(
xdev**2, makeTimes=0) / (GridUtil.getTimeSet(xdev).getLength() - 1)
ystddev = sumOverTime(
ydev**2, makeTimes=0) / (GridUtil.getTimeSet(ydev).getLength() - 1)
else:
raise VisADException(
"Number of timesteps for correlation should be same")
return noUnit(xydevsumbyn) / noUnit((xstddev**0.5) * (ystddev**0.5))
def wavefilter(tempF, minwave, maxwave):
if GridUtil.is3D(grid):
raise VisADException(
'This implementation of filtering supports only 2d,eg.., lat-lon grids'
)
tempFF = domainFactor(tempF, int(not (GridUtil.isLatLonOrder(tempF))))
for j, lat in enumerate(tempFF.domainEnumeration()):
fieldfft = fft(field(tempFF.evaluate(lat).getFloats()[0]))
reals = fieldfft.getFloats()[0]
imags = fieldfft.getFloats()[1]
if maxwave == None or int(maxwave) > len(reals):
maxwave = len(reals) - 1
for i in range(int(minwave), int(maxwave) + 1):
reals[i] = 0.0
imags[i] = 0.0
fieldfft.setSamples([reals, imags])
fieldifft = ifft(fieldfft)
tempFF[j].setSamples(GridUtil.getParam(fieldifft, 0).getFloats())
return tempFF.domainMultiply()
def stdMon(grid):
""" Create monthly standard deviations from a grid of monthly data over
a period of years. The number of timesteps must be a multiple of 12.
"""
from visad import VisADException
from visad import Gridded1DDoubleSet
from visad import FunctionType
from visad import FieldImpl
from visad import RealType
timeSet = GridUtil.getTimeSet(grid)
numT = timeSet.getLength()
if (numT % 12 > 0):
raise VisADException("Number of times must be a multiple of 12")
numYears = numT / 12
days = [[0, 31, 59, 90, 120, 151, 181, 212, 243, 273, 304, 334]]
units = [Util.parseUnit("days since 0001-01-01 00:00")]
newTimes = Gridded1DDoubleSet(RealType.Time, days, 12, None, units, None)
stdev = FieldImpl(FunctionType(RealType.Time,
grid.getSample(0).getType()), newTimes)
for month in range(12):
a = GridMath.applyFunctionOverTime(grid, GridMath.FUNC_STDEV, month,
12, 0)
stdev.setSample(month, a, 0)
return stdev
def getSampleTimeIndices(grid,
year=None,
season=None,
mon=None,
day=None,
hour=None,
min=None,
sec=None,
ms=None):
""" A Helper function to get indices a grid at specified time periods, multiple arguments can be used in
complex sampling. This function returns list of indices in grid.
"""
from visad import VisADException
searchformat = ""
searchstring = ""
if (str(mon) != "None" and str(season) != "None"):
raise VisADException("One of Month or Season can be used, not both")
if (str(year) != "None" and len(str(year)) == 4):
searchformat = searchformat + "yyyy"
searchstring = searchstring + str(year)
if (str(mon) != "None" and int(mon) in ([range(1, 13)])
and len(str(mon)) <= 2):
searchformat = searchformat + "MM"
if (len(str(mon)) < 2):
searchstring = searchstring + "0" + str(mon)
else:
searchstring = searchstring + str(mon)
if (str(day) != "None"):
searchformat = searchformat + "dd"
searchstring = searchstring + str(day)
if (str(hour) != "None"):
searchformat = searchformat + "HH"
searchstring = searchstring + str(hour)
if (str(min) != "None"):
searchformat = searchformat + "mm"
searchstring = searchstring + str(min)
if (str(sec) != "None"):
searchformat = searchformat + "ss"
searchstring = searchstring + str(sec)
if (str(ms) != "None"):
searchformat = searchformat + "ms"
searchstring = searchstring + str(ms)
if (str(season) != "None"):
seasons = ("djf", "jfm", "mam", "jja", "son", "ond", "jjas")
seasmons = ((12, 1, 2), (1, 2, 3), (3, 4, 5), (6, 7, 8), (9, 10, 11),
(10, 11, 12), (6, 7, 8, 9))
if (str(season).lower() in seasons):
montimes = getSampleTimesInFormat(grid, "MM")
alltimes = getSampleTimesInFormat(grid, searchformat.strip())
seasonlist = [list(seasons)]
seasonsearch = seasmons[list(seasons).index(str(season).lower())]
matchindices = [
i for i, t, m in zip(range(len(alltimes)), alltimes, montimes)
if t == searchstring and int(m) in seasonsearch
]
else:
raise VisADException("Season " + str(season) + " not found")
else:
alltimes = getSampleTimesInFormat(grid, searchformat.strip())
matchindices = [i for i, t in enumerate(alltimes) if t == searchstring]
return matchindices
def ddt(grid, timegradunit):
""" compute tendency (time derivative) using forward difference,
units of returned grid are units of grid per timegradient unit
timegradient unit can be month, day, hour, minute, seconds
"""
from visad import Real
from visad import FunctionType
from visad import FieldImpl
from visad import RealType
from visad import Gridded1DDoubleSet
from ucar.visad.data import CalendarDateTime
from visad import CommonUnit
from visad import VisADException
if (GridUtil.isTimeSequence(grid) == 1):
newTimeValues = []
timediffs = []
ts = GridUtil.getTimeSet(grid)
if (str(timegradunit).lower() in ("mon", "month", "months")):
timefactor = 86400.0 * 30
timegradunit = "month"
elif (str(timegradunit).lower() in ("day", "d", "days")):
timefactor = 86400.0
timegradunit = "day"
elif (str(timegradunit).lower() in ("hr", "hour", "hours")):
timefactor = 3600.0
timegradunit = "hr"
elif (str(timegradunit).lower() in ("m", "min", "minute", "minutes")):
timefactor = 60.0
timegradunit = "min"
elif (str(timegradunit).lower() in ("s", "sec", "second", "seconds")):
timefactor = 1.0
timegradunit = "s"
else:
raise VisADException(
"Requested time gradient unit is ambigious,use month,day,hour etc"
)
for i in range(grid.getDomainSet().getLength() - 1):
newTimeValues.append((ts[i].getValue() + ts[i + 1].getValue()) / 2)
prevtime = float(ts[i].getValue(CommonUnit.secondsSinceTheEpoch))
nexttime = float(ts[i + 1].getValue(
CommonUnit.secondsSinceTheEpoch))
timediffs.append((nexttime - prevtime) / timefactor)
newTimes = Gridded1DDoubleSet(RealType.Time, [newTimeValues],
len(newTimeValues), None,
ts.getSetUnits(), None)
ddtgrid = FieldImpl(
FunctionType(RealType.Time,
grid.getSample(0).getType()), newTimes)
for i in range(grid.getDomainSet().getLength() - 1):
diff = (grid.getSample(i + 1) - grid.getSample(i)).divide(
Real(timediffs[i]))
ddtgrid.setSample(i, diff)
unitname = str(
GridUtil.getParamType(grid).getComponent(0).getDefaultUnit())
print("[" + unitname + "]/" + str(timegradunit))
newunit = Util.parseUnit("(" + unitname + ")/" + str(timegradunit))
newType = Util.makeRealType("ddt of " + getVarName(grid), newunit)
else:
raise VisADException(
"Well, this data is not a time series, hard to do a time derivative!"
)
return GridUtil.setParamType(ddtgrid, newType, 0)
def createTimeMeans(grid, meanType="None"):
""" Create time mean of a grid at periods specified by type.
meanType can be yearly, monthly, daily, hourly, minutes, seconds
"""
from visad import Real
from visad import Gridded1DDoubleSet
from visad import FieldImpl
from visad import FunctionType
from visad import RealType
from visad import VisADException
if (str(meanType).lower() in ("year", "yr", "years", "yearly")):
searchFormat = "yyyy"
elif (str(meanType).lower() in ("mon", "month", "months", "monthly")):
searchFormat = "MM"
elif (str(meanType).lower() in ("day", "d", "days", "daily")):
searchFormat = "dd"
elif (str(meanType).lower() in ("hr", "hour", "hours", "hourly")):
searchFormat = "hh"
elif (str(meanType).lower()
in ("m", "min", "minute", "minutes", "minutely")):
searchFormat = "mm"
elif (str(meanType).lower() in ("s", "sec", "second", "seconds")):
searchFormat = "ss"
else:
raise VisADException(
"Unrecognized time mean type, use yearly or monthly etc")
alltimes = getSampleTimesInFormat(grid, searchFormat)
timeSet = GridUtil.getTimeSet(grid)
timeset = GridUtil.getTimeSet(grid).getSamples()[0]
timevalues = [i for i in timeset]
oldtime = alltimes[0]
temptime = 0
count = 0
newtimelist = []
for currt, tv, i in zip(alltimes, timevalues, range(len(alltimes))):
#values are always accumulated next time
if currt == oldtime:
#takes care of multiple times,first time,last time
temptime = temptime + tv
count = count + 1
if (i == (len(alltimes) - 1)):
newtimeval = temptime / count
newtimelist.append(newtimeval)
else:
#prev values are accumulated join to list
newtimeval = temptime / count
newtimelist.append(newtimeval)
count = 1
temptime = tv
oldtime = currt
if (i == (len(alltimes) - 1)):
newtimelist.append(temptime)
#create new time set
newTimes = Gridded1DDoubleSet(RealType.Time,
[newtimelist], len(newtimelist), None,
timeSet.getSetUnits(), None)
newdatalist = FieldImpl(
FunctionType(RealType.Time,
grid.getSample(0).getType()), newTimes)
timindices = range(len(newtimelist))
oldtime = alltimes[0]
tempdata = grid.getSample(0).multiply(Real(0.0))
count = 0
newind = 0
for currt, i in zip(alltimes, range(len(alltimes))):
#values are always accumulated next time
if currt == oldtime:
#takes care of multiple times,first time,last time
tempdata = tempdata.add(grid.getSample(i))
count = count + 1
if (i == (len(alltimes) - 1)):
newdatalist.setSample(newind, tempdata.divide(Real(count)))
newind = newind + 1
else:
#prev values are accumulated join to list
newdatalist.setSample(newind, tempdata.divide(Real(count)))
newind = newind + 1
count = 1
tempdata = grid.getSample(i)
oldtime = currt
if (i == (len(alltimes) - 1)):
newdatalist.setSample(newind, tempdata.divide(Real(count)))
newParamName = "Time Mean " + str(
Util.cleanTypeName(GridUtil.getParamType(grid)))
return newName(newdatalist, newParamName)