def accumulate24Hourly(data): """Returns 12-hourly data accumulated to 24-hours.""" newTimeValues=[] taxis=data.getTime() tunits=data.units print len(data.getTime()) newarray=[] for i in range((tlen/2)): p1=data(time=slice(i,i+1)) p2=data(time=slice(i+1,i+2)) accum=p1+p2 newarray.append(accum) newTimeValues.append(p2.getTime()[0]) array=MA.concatenate(newarray) array=MA.array(array, 'f', fill_value=data.getMissing()) axes=data.getAxisList() newTimeAxis=cdms.createAxis(newTimeValues) newTimeAxis.units=tunits newTimeAxis.designateTime() newTimeAxis.id=newTimeAxis.long_name=newTimeAxis.title="time" newaxes=[newTimeAxis]+axes[1:] var=cdms.createVariable(array, axes=newaxes, id=data.id) for att in ("units", "long_name"): setattr(var, att, getattr(data, att)) return var
def flatten2DTimeData(var, time_var): """ Returns a flattened 2D array variable with a recalculated time axies. """ data = MV.ravel(var) missing = var.getMissing() time_values = time_var._data time_unit = time_var.units sampling_rate = var.getAxis(1) rate = int(sampling_rate.id[3:]) newtime_values = [] for t in time_values: for i in range(rate): tvalue = t + ((1./rate)*i) newtime_values.append(tvalue) new_time_ax = cdms.createAxis(newtime_values) new_time_ax.units = time_unit new_time_ax.designateTime() new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time" atts = var.attributes newAtts = {} for att,value in atts.items(): if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value) == 1: value = value[0] if type(value) in (type(1), type(1.), type(long(1))): newAtts[att] = value elif type(value) == type("string"): newAtts[att] = value.strip() # Now create the variable newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts) return newvar
def _recreateVariable(self, var, newArray, timeAxis, flagVar, flagMax, missingValue, sampleBy="averaging"): """ Rebuilds a variable using new array and new time axis. """ atts=var.attributes newAtts={} for att,value in atts.items(): if type(value) in (type((1,1)), type([1,2]), type(Numeric.array([1.]))) and len(value)==1: value=value[0] if type(value) in (type(1), type(1.), type(long(1))): newAtts[att]=value elif type(value)==type(""): newAtts[att]=value.strip() # Now create the variable missing=missingValue data=newArray newvar=cdms.createVariable(data, id=var.id, fill_value=missing, axes=[timeAxis], attributes=newAtts) newvar.frequency=1 if sampleBy=="averaging": newvar.comment="Data averaged to give a pseudo sampling frequency of 1 Hz. " if flagVar: newvar.comment=newvar.comment+"Values were included where the flag variable was less than %s and where both the flag value and actual data value were present (i.e. not missing values)." % (flagMax+1) newvar.comment=newvar.comment+" Average data values were set to missing where more than half the values in any one second period were already missing. Missing value was also used where more than half the flags were either missing value or values of 2 or greater." else: newvar.comment=newvar.comment+"The flag variable was not used in creating this variable." newvar.comment=newvar.comment+" Average data values were set to missing where more than half the values in any one second period were already missing." elif sampleBy=="selection": newvar.comment="Data sub-sampled to give a pseudo sampling frequency of 1 Hz by selecting only the per-second frequency values. " if flagVar: newvar.comment=newvar.comment+" Values were included where the flag variable was less than 2 and where both the flag value and actual data value were present (i.e. not missing values)." else: newvar.comment=newvar.comment+" The flag variable was not used in creating this variable." return newvar
def _recreateVariable(self, var, newArray, timeAxis, flagVar, flagMax, missingValue, sampleBy="averaging"): """ Rebuilds a variable using new array and new time axis. """ atts = var.attributes newAtts = {} for att, value in atts.items(): if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array([1.0]))) and len(value) == 1: value = value[0] if type(value) in (type(1), type(1.0), type(long(1))): newAtts[att] = value elif type(value) == type(""): newAtts[att] = value.strip() # Now create the variable missing = missingValue data = newArray newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[timeAxis], attributes=newAtts) newvar.frequency = 1 if sampleBy == "averaging": newvar.comment = "Data averaged to give a pseudo sampling frequency of 1 Hz. " if flagVar: newvar.comment = ( newvar.comment + "Values were included where the flag variable was less than %s and where both the flag value and actual data value were present (i.e. not missing values)." % (flagMax + 1) ) newvar.comment = ( newvar.comment + " Average data values were set to missing where more than half the values in any one second period were already missing. Missing value was also used where more than half the flags were either missing value or values of 2 or greater." ) else: newvar.comment = newvar.comment + "The flag variable was not used in creating this variable." newvar.comment = ( newvar.comment + " Average data values were set to missing where more than half the values in any one second period were already missing." ) elif sampleBy == "selection": newvar.comment = "Data sub-sampled to give a pseudo sampling frequency of 1 Hz by selecting only the per-second frequency values. " if flagVar: newvar.comment = ( newvar.comment + " Values were included where the flag variable was less than 2 and where both the flag value and actual data value were present (i.e. not missing values)." ) else: newvar.comment = newvar.comment + " The flag variable was not used in creating this variable." return newvar
def flatten2DTimeData(var, time_var): """ Returns a flattened 2D array variable with a recalculated time axies. """ data = MV.ravel(var) missing = var.getMissing() time_values = time_var._data time_unit = time_var.units sampling_rate = var.getAxis(1) rate = int(sampling_rate.id[3:]) newtime_values = [] for t in time_values: for i in range(rate): tvalue = t + ((1. / rate) * i) newtime_values.append(tvalue) new_time_ax = cdms.createAxis(newtime_values) new_time_ax.units = time_unit new_time_ax.designateTime() new_time_ax.id = new_time_ax.long_name = new_time_ax.standard_name = "time" atts = var.attributes newAtts = {} for att, value in atts.items(): if type(value) in (type((1, 1)), type([1, 2]), type(Numeric.array( [1.]))) and len(value) == 1: value = value[0] if type(value) in (type(1), type(1.), type(long(1))): newAtts[att] = value elif type(value) == type("string"): newAtts[att] = value.strip() # Now create the variable newvar = cdms.createVariable(data, id=var.id, fill_value=missing, axes=[new_time_ax], attributes=newAtts) return newvar
def writeOutput(infile, var, lat, lon, dav, dmax, dmin, sdupper, sdlower, location): """ Writes an output file of the variables generated. """ location=location.replace(" ","_").lower() f=cdms.open(infile) mapit={"apcp":("rainfall","l/m^2"), "tmpk":("temperature","K")} varname=mapit[var][0] units=mapit[var][1] datetime=os.path.split(infile)[-1].split(".")[1] outfile="%s_%s_%s.nc" % (datetime, location, varname) outpath=os.path.split(infile)[0] outfile=os.path.join(outpath, outfile) fout=cdms.open(outfile, "w") latax=cdms.createAxis([lat]) latax.units="degrees_north" latax.id=latax.standard_name=latax.long_name="latitude" lonax=cdms.createAxis([lon]) lonax.units="degrees_east" lonax.id=lonax.standard_name=lonax.long_name="longitude" tax=f[var].getTime() #f(var, level=slice(0,1), lat=slice(0,1), lon=slice(0,1)).getTime() timeax=cdms.createAxis(Numeric.array(tax[0:tlen],'d')) timeax.designateTime() timeax.units=tax.units #timeax.id=timeax.standard_name=timeax.long_name="time" timeax.id="time" timeax.title=tax.title timeax.delta_t=tax.delta_t timeax.init_time=tax.init_time timeax.actual_range=tax.actual_range del timeax.axis del timeax.calendar metadata=f[var] fv=metadata.missing_value newshape=(len(timeax), len(latax), len(lonax)) maxFound=20. # Set as our max value if not greater for v in ((dav, "average"), (dmax, "maximum"), (dmin, "minimum"), \ (sdupper, "plus_std_dev"), (sdlower, "minus_std_dev"), ("always10", "always10")): if type(v[0])==type("jlj") and v[0]=="always10": print "Creating always equal to 10 variable." always10=MA.zeros(newshape, 'f')+10. #print always10.shape, dav.shape, type(dav) newvar=cdms.createVariable(always10, axes=[timeax, latax, lonax], id=v[1], fill_value=fv) newvar.longname="always10" else: data=v[0] name=varname+"_"+v[1] if not type(data)==type([1,2]): data=data(squeeze=1) data=MA.resize(data, newshape) newvar=cdms.createVariable(data, axes=[timeax, latax, lonax], id=name, fill_value=fv) newvar.long_name="%s - %s" % (varname.title(), v[1].replace("_", " ")) newvar.units=metadata.units (dummy,vmax)=vcs.minmax(newvar) if vmax>maxFound: maxFound=vmax fout.write(newvar) fout.sync() del newvar fout.close() return (outfile, varname, datetime, maxFound)
def cdms2na(ncfile, na_file_names, naVars={}, variables=None, nFilesOnly="no", rule=None, ffi="automatic", delimiter=" ", float_format="%g", rules=None, sizeLimit=None): """ Main conversion function that calls the appropriate classes and functions to write a NASA Ames file. """ #print infilename, outfilenames, nFilesOnly, naVars, variables if type(na_file_names) == type("string"): na_file_names = [na_file_names] # Get which NASA Ames internal variables are allowed to be overwritten in the output files (i.e. by user inputs) allowedOverwriteMetadata = ("DATE", "RDATE", "ANAME", "MNAME", "ONAME", "ORG", "SNAME", "VNAME") arrayArgs=["DATE", "RDATE", "ANAME", "VNAME"] # ANAME[a] - array of 'a' x ANAME strings - aux var names # DATE (array of three) - UT date at which the data within the file starts # MNAME - mission name # ONAME - name of originator(s) # ORG - org or affiliation of originator(s) # RDATE (array of three) - date of data reduction or revision # SNAME - source of measurement or model output VNAME[n] - array of 'n' x # VNAME strings - var names. outputMessage=[] msg="Reading data from: %s\n" % infilename print msg outputMessage.append(msg) cdmsfile=cdms.open(infilename) globals=cdmsfile.attributes vars=[] if not variables: variables=cdmsfile.listvariables() #for var in cdmsfile.listvariables(): #vars.append(cdmsfile(var)) for variable in variables: varObj=cdmsfile(variable) # Deal with singleton variables if not hasattr(varObj, "rank"): varMetadata=cdmsfile[variable].attributes varValue=varObj #print varMetadata, varValue, varMetadata.keys(), varMetadata._obj_.id varObj=cdms.createVariable(Numeric.array(varObj), id=getBestName(varMetadata).replace(" ", "_"), attributes=varMetadata) #print varObj, dir(varObj); sys.exit() varObj.value=varObj._data[0] #varObj.rank=0 #print varObj, varObj.attributes vars.append(varObj) # Re-order variables if they have the attribute 'nasa_ames_var_number' orderedVars=[None]*1000 otherVars=[] for var in vars: varMetadata=cdmsfile[var] if hasattr(varMetadata, "nasa_ames_var_number"): num=varMetadata.nasa_ames_var_number orderedVars[num]=var else: otherVars.append(var) vars=[] for var in orderedVars: if var!=None: vars.append(var) vars=vars+otherVars builder=NAContentCollector(vars, globals, rule=rule, cdmsfile=cdmsfile) #print builder.na_dict["X"] builtNADicts=[[builder.na_dict, builder.varIDs]] if builder.varIDs==None: msg="\nNo files created after variables parsed." print msg outputMessage.append(msg) return outputMessage while len(builder.varBin)>0: builder=NAContentCollector(builder.varBin, globals, rule=rule, cdmsfile=cdmsfile) outputMessage=outputMessage+builder.outputMessage if builder.varIDs!=None: builtNADicts.append([builder.na_dict, builder.varIDs]) # Return only filenames if only want to know them now. ncount=1 fileNames=[] if nFilesOnly=="yes": for i in builtNADicts: if len(builtNADicts)==1: suffix="" else: suffix="_%s" % ncount nameparts=outfilenames[0].split(".") newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1] fileNames.append(newname) ncount=ncount+1 return fileNames msg="\n%s files to write" % len(builtNADicts) print msg outputMessage.append(msg) count=1 ncount=1 for i in builtNADicts: if len(outfilenames)==1: if len(builtNADicts)==1: suffix="" else: suffix="_%s" % ncount nameparts=outfilenames[0].split(".") newname=(".".join(nameparts[:-1]))+suffix+"."+nameparts[-1] else: newname=outfilenames[count-1] msg="\nWriting output NASA Ames file: %s" % newname print msg outputMessage.append(msg) builtNADict=i[0] for key in naVars.keys(): if key in allowedOverwriteMetadata: if key in arrayArgs: newItem=naVars[key].split() else: newItem=naVars[key] if newItem!=builtNADict[key]: builtNADict[key]=newItem msg="Metadata overwritten in output file: '%s' is now '%s'" % (key, builtNADict[key]) print msg outputMessage.append(msg) fileList=[] # Cope with size limits if specified and FFI is 1001 if sizeLimit and (builtNADict["FFI"]==1001 and len(builtNADict["V"][0])>sizeLimit): varList=builtNADict["V"] arrayLength=len(varList[0]) nvolInfo=divmod(arrayLength, sizeLimit) nvol=nvolInfo[0] if nvolInfo[1]>0: nvol=nvol+1 start=0 letterCount=0 ivol=0 while start<arrayLength: ivol=ivol+1 end=start+sizeLimit if end>arrayLength: end=arrayLength currentBlock=[] # Write new V array for v in varList: currentBlock.append(v[start:end]) # Adjust X accordingly NADictCopy=modifyNADictCopy(builtNADict, currentBlock, start, end, ivol, nvol) # Write data to output file newnamePlusLetter="%s-%.3d.na" % (newname[:-3], ivol) fileList.append(newnamePlusLetter) general.openNAFile(newnamePlusLetter, 'w', NADictCopy, delimiter=delimiter, float_format=float_format) msg="\nOutput files split on size limit: %s\nFilename used: %s" % (sizeLimit, newnamePlusLetter) print msg outputMessage.append(msg) letterCount=letterCount+1 start=end else: general.openNAFile(newname, 'w', builtNADict, delimiter=delimiter, float_format=float_format) msg="\nWrote the following variables:"+"\n\t"+("\n\t".join(i[1][0])) print msg outputMessage.append(msg) if len(i[1][1])>0: msg="\nWrote the following auxiliary variables:" msg=msg+"\n\t"+("\n\t".join(i[1][1])) if len(i[1][2])>0: msg="\nWrote the following Singleton variables:" msg=msg+"\n\t"+("\n\t".join(i[1][2])) if len(fileList)>0: msg=msg+("\n\nNASA Ames files written successfully: \n%s" % "\n".join(fileList)) count=count+len(fileList) else: msg=msg+"\n\nNASA Ames file written successfully: %s" % newname count=count+1 ncount=ncount+1 print msg outputMessage.append(msg) if (count-1)==1: plural="" else: plural="s" msg="\n%s file%s written." % ((count-1), plural) print msg outputMessage.append(msg) return outputMessage
#!/usr/bin/env python import cdms, vcs, cdutil, genutil, os, sys, Numeric from cdms import MV f_in1 = cdms.open('./pot_temp_geopotential.nc') f_in2 = cdms.open('./press_rho_temp.nc') print print "Reading in variables" theta = f_in1('theta') geopotential = f_in1('geopotential') pl = f_in2('pl') print print "Truncating variables" theta = cdms.createVariable(theta[:, :, 0, :], copy=1) geopotential = cdms.createVariable(geopotential[:, :, 0, :], copy=1) pl = cdms.createVariable(pl[:, :, 0, :], copy=1) print print "Calculating vpgf" exec(open('calc_vpgf.txt')) f_out = cdms.open('./vpgf.nc', 'w') print print "Writing vpgf to file" f_out.write(vpgf) print print "Done"
end_time = cdtime.comptime(1993,12,1) ac1=cdutil.ANNUALCYCLE.climatology(data1(time=(start_time, end_time, 'cob'))) ac2=cdutil.ANNUALCYCLE.climatology(data2(time=(start_time, end_time, 'cob'))) print ac1 data1=cdutil.ANNUALCYCLE.departures(data1,ref=ac1) data2=cdutil.ANNUALCYCLE.departures(data2,ref=ac2) print data1.shape,data2.shape tim = data2.getTime() lat=data2.getLatitude() lon=data2.getLongitude() data1=cdms.createVariable(data1,axes=[tim,lat,lon],typecode='f',id='tas') diff=MV.subtract(data1,data2) # zonal differences z_diff=MV.average(diff,2) print 'Zonal data shape (before): ',z_diff.shape z_diff=MV.transpose(z_diff,(1,0)) # add id to data z_diff.id='zonal_diff' print 'Zonal data shape (after): ',z_diff.shape # global differences gl_diff=cdutil.averager(diff,axis='xy')
#a=cdms.open('/pcmdi/AMIP3/amip/mo/'+var+'/'+model[i]+'/'+var+'_'+model[i]+'.xml') a=cdms.open(file_xml) data=a(var,time=(start_time,end_time),squeeze=1) a.close() ac=cdutil.ANNUALCYCLE.climatology(data(time=(start_time, end_time, 'cob'))) data_an=cdutil.ANNUALCYCLE.departures(data,ref=ac) print i,model[i],data.shape, data_an.shape glan[i,:]=cdutil.averager(data_an,axis='xy') # # setup metadata and write out to a netcdf file # tim=data.getTime() runs=Numeric.arange(0,len(model)) runs=cdms.createAxis(runs,id='models') glan=cdms.createVariable(glan,axes=(runs,tim),id='global_'+var+'_anomalies') # print 'start write file..' q=cdms.open('global_anomalies.nc','w') q.model_designation=model_description q.write(glan) q.close() # # a simple plot # x=vcs.init() x.setcolormap('default') x.plot(glan) # print "" print "Press the Return key to finish."