def accumulate24Hourly(data):
"""Returns 12-hourly data accumulated to 24-hours."""
newTimeValues=[]
taxis=data.getTime()
tunits=data.units
print len(data.getTime())
newarray=[]
for i in range((tlen/2)):
p1=data(time=slice(i,i+1))
p2=data(time=slice(i+1,i+2))
accum=p1+p2
newarray.append(accum)
newTimeValues.append(p2.getTime()[0])
array=MA.concatenate(newarray)
array=MA.array(array, 'f', fill_value=data.getMissing())
axes=data.getAxisList()
newTimeAxis=cdms.createAxis(newTimeValues)
newTimeAxis.units=tunits
newTimeAxis.designateTime()
newTimeAxis.id=newTimeAxis.long_name=newTimeAxis.title="time"
newaxes=[newTimeAxis]+axes[1:]
var=cdms.createVariable(array, axes=newaxes, id=data.id)
for att in ("units", "long_name"):
setattr(var, att, getattr(data, att))
return var
def flatten2DTimeData(var, time_var):
"""
Returns a flattened 2D array variable with a recalculated time axies.
"""
data = MV.ravel(var)
missing = var.getMissing()
time_values = time_var._data
time_unit = time_var.units
sampling_rate = var.getAxis(1)
rate = int(sampling_rate.id[3:])
newtime_values = []
for t in time_values:
for i in range(rate):
tvalue = t + ((1./rate)*i)
newtime_values.append(tvalue)
new_time_ax = cdms.createAxis(newtime_values)
new_time_ax.units = time_unit
new_time_ax.designateTime()
new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time"
atts = var.attributes
newAtts = {}
for att,value in atts.items():
if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value) == 1:
value = value[0]
if type(value) in (type(1), type(1.), type(long(1))):
newAtts[att] = value
elif type(value) == type("string"):
newAtts[att] = value.strip()
# Now create the variable
newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts)
return newvar
def _recreateVariable(self, var, newArray, timeAxis, flagVar, flagMax, missingValue, sampleBy="averaging"):
"""
Rebuilds a variable using new array and new time axis.
"""
atts=var.attributes
newAtts={}
for att,value in atts.items():
if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value)==1:
value=value[0]
if type(value) in (type(1), type(1.), type(long(1))):
newAtts[att]=value
elif type(value)==type(""):
newAtts[att]=value.strip()
# Now create the variable
missing=missingValue
data=newArray
newvar=cdms.createVariable(data, id=var.id, fill_value=missing, axes=[timeAxis], attributes=newAtts)
newvar.frequency=1
if sampleBy=="averaging":
newvar.comment="Data averaged to give a pseudo sampling frequency of 1 Hz. "
if flagVar:
newvar.comment=newvar.comment+"Values were included where the flag variable was less than %s and where both the flag value and actual data value were present (i.e. not missing values)." % (flagMax+1)
newvar.comment=newvar.comment+" Average data values were set to missing where more than half the values in any one second period were already missing. Missing value was also used where more than half the flags were either missing value or values of 2 or greater."
else:
newvar.comment=newvar.comment+"The flag variable was not used in creating this variable."
newvar.comment=newvar.comment+" Average data values were set to missing where more than half the values in any one second period were already missing."
elif sampleBy=="selection":
newvar.comment="Data sub-sampled to give a pseudo sampling frequency of 1 Hz by selecting only the per-second frequency values. "
if flagVar:
newvar.comment=newvar.comment+" Values were included where the flag variable was less than 2 and where both the flag value and actual data value were present (i.e. not missing values)."
else:
newvar.comment=newvar.comment+" The flag variable was not used in creating this variable."
return newvar
def _recreateVariable(self, var, newArray, timeAxis, flagVar, flagMax, missingValue, sampleBy="averaging"):
"""
Rebuilds a variable using new array and new time axis.
"""
atts = var.attributes
newAtts = {}
for att, value in atts.items():
if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array([1.0]))) and len(value) == 1:
value = value[0]
if type(value) in (type(1), type(1.0), type(long(1))):
newAtts[att] = value
elif type(value) == type(""):
newAtts[att] = value.strip()
# Now create the variable
missing = missingValue
data = newArray
newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[timeAxis], attributes=newAtts)
newvar.frequency = 1
if sampleBy == "averaging":
newvar.comment = "Data averaged to give a pseudo sampling frequency of 1 Hz. "
if flagVar:
newvar.comment = (
newvar.comment
+ "Values were included where the flag variable was less than %s and where both the flag value and actual data value were present (i.e. not missing values)."
% (flagMax + 1)
)
newvar.comment = (
newvar.comment
+ " Average data values were set to missing where more than half the values in any one second period were already missing. Missing value was also used where more than half the flags were either missing value or values of 2 or greater."
)
else:
newvar.comment = newvar.comment + "The flag variable was not used in creating this variable."
newvar.comment = (
newvar.comment
+ " Average data values were set to missing where more than half the values in any one second period were already missing."
)
elif sampleBy == "selection":
newvar.comment = "Data sub-sampled to give a pseudo sampling frequency of 1 Hz by selecting only the per-second frequency values. "
if flagVar:
newvar.comment = (
newvar.comment
+ " Values were included where the flag variable was less than 2 and where both the flag value and actual data value were present (i.e. not missing values)."
)
else:
newvar.comment = newvar.comment + " The flag variable was not used in creating this variable."
return newvar
def flatten2DTimeData(var, time_var):
"""
Returns a flattened 2D array variable with a recalculated time axies.
"""
data = MV.ravel(var)
missing = var.getMissing()
time_values = time_var._data
time_unit = time_var.units
sampling_rate = var.getAxis(1)
rate = int(sampling_rate.id[3:])
newtime_values = []
for t in time_values:
for i in range(rate):
tvalue = t + ((1. / rate) * i)
newtime_values.append(tvalue)
new_time_ax = cdms.createAxis(newtime_values)
new_time_ax.units = time_unit
new_time_ax.designateTime()
new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time"
atts = var.attributes
newAtts = {}
for att, value in atts.items():
if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array(
[1.]))) and len(value) == 1:
value = value[0]
if type(value) in (type(1), type(1.), type(long(1))):
newAtts[att] = value
elif type(value) == type("string"):
newAtts[att] = value.strip()
# Now create the variable
newvar = cdms.createVariable(data,
id=var.id,
fill_value=missing,
axes=[new_time_ax],
attributes=newAtts)
return newvar
def writeOutput(infile, var, lat, lon, dav, dmax, dmin, sdupper, sdlower, location):
"""
Writes an output file of the variables generated.
"""
location=location.replace(" ","_").lower()
f=cdms.open(infile)
mapit={"apcp":("rainfall","l/m^2"), "tmpk":("temperature","K")}
varname=mapit[var][0]
units=mapit[var][1]
datetime=os.path.split(infile)[-1].split(".")[1]
outfile="%s_%s_%s.nc" % (datetime, location, varname)
outpath=os.path.split(infile)[0]
outfile=os.path.join(outpath, outfile)
fout=cdms.open(outfile, "w")
latax=cdms.createAxis([lat])
latax.units="degrees_north"
latax.id=latax.standard_name=latax.long_name="latitude"
lonax=cdms.createAxis([lon])
lonax.units="degrees_east"
lonax.id=lonax.standard_name=lonax.long_name="longitude"
tax=f[var].getTime() #f(var, level=slice(0,1), lat=slice(0,1), lon=slice(0,1)).getTime()
timeax=cdms.createAxis(Numeric.array(tax[0:tlen],'d'))
timeax.designateTime()
timeax.units=tax.units
#timeax.id=timeax.standard_name=timeax.long_name="time"
timeax.id="time"
timeax.title=tax.title
timeax.delta_t=tax.delta_t
timeax.init_time=tax.init_time
timeax.actual_range=tax.actual_range
del timeax.axis
del timeax.calendar
metadata=f[var]
fv=metadata.missing_value
newshape=(len(timeax), len(latax), len(lonax))
maxFound=20. # Set as our max value if not greater
for v in ((dav, "average"), (dmax, "maximum"), (dmin, "minimum"), \
(sdupper, "plus_std_dev"), (sdlower, "minus_std_dev"), ("always10", "always10")):
if type(v[0])==type("jlj") and v[0]=="always10":
print "Creating always equal to 10 variable."
always10=MA.zeros(newshape, 'f')+10.
#print always10.shape, dav.shape, type(dav)
newvar=cdms.createVariable(always10, axes=[timeax, latax, lonax], id=v[1], fill_value=fv)
newvar.longname="always10"
else:
data=v[0]
name=varname+"_"+v[1]
if not type(data)==type([1,2]):
data=data(squeeze=1)
data=MA.resize(data, newshape)
newvar=cdms.createVariable(data, axes=[timeax, latax, lonax], id=name, fill_value=fv)
newvar.long_name="%s - %s" % (varname.title(), v[1].replace("_", " "))
newvar.units=metadata.units
(dummy,vmax)=vcs.minmax(newvar)
if vmax>maxFound:
maxFound=vmax
fout.write(newvar)
fout.sync()
del newvar
fout.close()
return (outfile, varname, datetime, maxFound)
def cdms2na(ncfile, na_file_names, naVars={}, variables=None, nFilesOnly="no",
rule=None, ffi="automatic", delimiter=" ", float_format="%g",
rules=None, sizeLimit=None):
"""
Main conversion function that calls the appropriate classes and functions
to write a NASA Ames file.
"""
#print infilename, outfilenames, nFilesOnly, naVars, variables
if type(na_file_names) == type("string"):
na_file_names = [na_file_names]
# Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs)
allowedOverwriteMetadata = ("DATE", "RDATE", "ANAME", "MNAME",
"ONAME", "ORG", "SNAME", "VNAME")
arrayArgs=["DATE", "RDATE", "ANAME", "VNAME"]
# ANAME[a] - array of 'a' x ANAME strings - aux var names
# DATE (array of three) - UT date at which the data within the file starts
# MNAME - mission name
# ONAME - name of originator(s)
# ORG - org or affiliation of originator(s)
# RDATE (array of three) - date of data reduction or revision
# SNAME - source of measurement or model output VNAME[n] - array of 'n' x
# VNAME strings - var names.
outputMessage=[]
msg="Reading data from: %s\n" % infilename
print msg
outputMessage.append(msg)
cdmsfile=cdms.open(infilename)
globals=cdmsfile.attributes
vars=[]
if not variables:
variables=cdmsfile.listvariables()
#for var in cdmsfile.listvariables():
#vars.append(cdmsfile(var))
for variable in variables:
varObj=cdmsfile(variable)
# Deal with singleton variables
if not hasattr(varObj, "rank"):
varMetadata=cdmsfile[variable].attributes
varValue=varObj
#print varMetadata, varValue, varMetadata.keys(), varMetadata._obj_.id
varObj=cdms.createVariable(Numeric.array(varObj), id=getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata)
#print varObj, dir(varObj); sys.exit()
varObj.value=varObj._data[0]
#varObj.rank=0
#print varObj, varObj.attributes
vars.append(varObj)
# Re-order variables if they have the attribute 'nasa_ames_var_number'
orderedVars=[None]*1000
otherVars=[]
for var in vars:
varMetadata=cdmsfile[var]
if hasattr(varMetadata, "nasa_ames_var_number"):
num=varMetadata.nasa_ames_var_number
orderedVars[num]=var
else:
otherVars.append(var)
vars=[]
for var in orderedVars:
if var!=None:
vars.append(var)
vars=vars+otherVars
builder=NAContentCollector(vars, globals, rule=rule, cdmsfile=cdmsfile)
#print builder.na_dict["X"]
builtNADicts=[[builder.na_dict, builder.varIDs]]
if builder.varIDs==None:
msg="\nNo files created after variables parsed."
print msg
outputMessage.append(msg)
return outputMessage
while len(builder.varBin)>0:
builder=NAContentCollector(builder.varBin, globals, rule=rule, cdmsfile=cdmsfile)
outputMessage=outputMessage+builder.outputMessage
if builder.varIDs!=None: builtNADicts.append([builder.na_dict, builder.varIDs])
# Return only filenames if only want to know them now.
ncount=1
fileNames=[]
if nFilesOnly=="yes":
for i in builtNADicts:
if len(builtNADicts)==1:
suffix=""
else:
suffix="_%s" % ncount
nameparts=outfilenames[0].split(".")
newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1]
fileNames.append(newname)
ncount=ncount+1
return fileNames
msg="\n%s files to write" % len(builtNADicts)
print msg
outputMessage.append(msg)
count=1
ncount=1
for i in builtNADicts:
if len(outfilenames)==1:
if len(builtNADicts)==1:
suffix=""
else:
suffix="_%s" % ncount
nameparts=outfilenames[0].split(".")
newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1]
else:
newname=outfilenames[count-1]
msg="\nWriting output NASA Ames file: %s" % newname
print msg
outputMessage.append(msg)
builtNADict=i[0]
for key in naVars.keys():
if key in allowedOverwriteMetadata:
if key in arrayArgs:
newItem=naVars[key].split()
else:
newItem=naVars[key]
if newItem!=builtNADict[key]:
builtNADict[key]=newItem
msg="Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key])
print msg
outputMessage.append(msg)
fileList=[]
# Cope with size limits if specified and FFI is 1001
if sizeLimit and (builtNADict["FFI"]==1001 and len(builtNADict["V"][0])>sizeLimit):
varList=builtNADict["V"]
arrayLength=len(varList[0])
nvolInfo=divmod(arrayLength, sizeLimit)
nvol=nvolInfo[0]
if nvolInfo[1]>0: nvol=nvol+1
start=0
letterCount=0
ivol=0
while start<arrayLength:
ivol=ivol+1
end=start+sizeLimit
if end>arrayLength:
end=arrayLength
currentBlock=[]
# Write new V array
for v in varList:
currentBlock.append(v[start:end])
# Adjust X accordingly
NADictCopy=modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol)
# Write data to output file
newnamePlusLetter="%s-%.3d.na" % (newname[:-3], ivol)
fileList.append(newnamePlusLetter)
general.openNAFile(newnamePlusLetter, 'w', NADictCopy, delimiter=delimiter, float_format=float_format)
msg="\nOutput files split on size limit: %s\nFilename used: %s" % (sizeLimit, newnamePlusLetter)
print msg
outputMessage.append(msg)
letterCount=letterCount+1
start=end
else:
general.openNAFile(newname, 'w', builtNADict, delimiter=delimiter, float_format=float_format)
msg="\nWrote the following variables:"+"\n\t"+("\n\t".join(i[1][0]))
print msg
outputMessage.append(msg)
if len(i[1][1])>0:
msg="\nWrote the following auxiliary variables:"
msg=msg+"\n\t"+("\n\t".join(i[1][1]))
if len(i[1][2])>0:
msg="\nWrote the following Singleton variables:"
msg=msg+"\n\t"+("\n\t".join(i[1][2]))
if len(fileList)>0:
msg=msg+("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList))
count=count+len(fileList)
else:
msg=msg+"\n\nNASA Ames file written successfully: %s" % newname
count=count+1
ncount=ncount+1
print msg
outputMessage.append(msg)
if (count-1)==1:
plural=""
else:
plural="s"
msg="\n%s file%s written." % ((count-1), plural)
print msg
outputMessage.append(msg)
return outputMessage
#!/usr/bin/env python
import cdms, vcs, cdutil, genutil, os, sys, Numeric
from cdms import MV
f_in1 = cdms.open('./pot_temp_geopotential.nc')
f_in2 = cdms.open('./press_rho_temp.nc')
print
print "Reading in variables"
theta = f_in1('theta')
geopotential = f_in1('geopotential')
pl = f_in2('pl')
print
print "Truncating variables"
theta = cdms.createVariable(theta[:, :, 0, :], copy=1)
geopotential = cdms.createVariable(geopotential[:, :, 0, :], copy=1)
pl = cdms.createVariable(pl[:, :, 0, :], copy=1)
print
print "Calculating vpgf"
exec(open('calc_vpgf.txt'))
f_out = cdms.open('./vpgf.nc', 'w')
print
print "Writing vpgf to file"
f_out.write(vpgf)
print
print "Done"
end_time = cdtime.comptime(1993,12,1) ac1=cdutil.ANNUALCYCLE.climatology(data1(time=(start_time, end_time, 'cob'))) ac2=cdutil.ANNUALCYCLE.climatology(data2(time=(start_time, end_time, 'cob'))) print ac1 data1=cdutil.ANNUALCYCLE.departures(data1,ref=ac1) data2=cdutil.ANNUALCYCLE.departures(data2,ref=ac2) print data1.shape,data2.shape tim = data2.getTime() lat=data2.getLatitude() lon=data2.getLongitude() data1=cdms.createVariable(data1,axes=[tim,lat,lon],typecode='f',id='tas') diff=MV.subtract(data1,data2) # zonal differences z_diff=MV.average(diff,2) print 'Zonal data shape (before): ',z_diff.shape z_diff=MV.transpose(z_diff,(1,0)) # add id to data z_diff.id='zonal_diff' print 'Zonal data shape (after): ',z_diff.shape # global differences gl_diff=cdutil.averager(diff,axis='xy')
#a=cdms.open('/pcmdi/AMIP3/amip/mo/'+var+'/'+model[i]+'/'+var+'_'+model[i]+'.xml')
a=cdms.open(file_xml)
data=a(var,time=(start_time,end_time),squeeze=1)
a.close()
ac=cdutil.ANNUALCYCLE.climatology(data(time=(start_time, end_time, 'cob')))
data_an=cdutil.ANNUALCYCLE.departures(data,ref=ac)
print i,model[i],data.shape, data_an.shape
glan[i,:]=cdutil.averager(data_an,axis='xy')
#
# setup metadata and write out to a netcdf file
#
tim=data.getTime()
runs=Numeric.arange(0,len(model))
runs=cdms.createAxis(runs,id='models')
glan=cdms.createVariable(glan,axes=(runs,tim),id='global_'+var+'_anomalies')
#
print 'start write file..'
q=cdms.open('global_anomalies.nc','w')
q.model_designation=model_description
q.write(glan)
q.close()
#
# a simple plot
#
x=vcs.init()
x.setcolormap('default')
x.plot(glan)
#
print ""
print "Press the Return key to finish."
