def __init__(self, fpatt, layout=None, multitime=False):
fnames = glob.glob(fpatt)
# if multitime:
# iters = [ f[-18:-8] for f in fnames if f.endswith('.t001.nc') ]
# iters.sort()
# fnames_first = [ f for f in fnames if f[-18:-8] == iters[0] ]
# else:
# fnames_first = fnames
fnames.sort()
# open files
self.nc = [netcdf_file(f, 'r') for f in fnames]
# global attributes
# get from first file, but remove/reset tile-specific ones
self._attributes = getattributes(self.nc[0], _exclude_global)
self._attributes['tile_number'] = 1
self._attributes['bi'] = 1
self._attributes['bj'] = 1
haveexch2 = False
for k in self._attributes.keys():
if k.startswith('exch2_'):
del self._attributes[k]
haveexch2 = True
sNx = self.sNx
sNy = self.sNy
ntx = self.nSx * self.nPx
nty = self.nSy * self.nPy
if layout is None:
if haveexch2:
layout = 'exch2'
else:
layout = 'model'
self.layout = layout
# precompute indices
self._i0 = []
self._ie = []
self._j0 = []
self._je = []
self._fn = []
self._nf = 0
if layout == 'model':
self._nx = self.Nx
self._ny = self.Ny
for nc in self.nc:
tn = nc.tile_number
bj, bi = divmod(tn - 1, ntx)
ie = sNx * (bi + 1 - ntx)
je = sNy * (bj + 1 - nty)
self._i0.append(sNx * bi)
self._j0.append(sNy * bj)
self._ie.append(ie or None)
self._je.append(je or None)
elif layout == 'exch2':
self._nx = 0
self._ny = 0
for nc in self.nc:
i0 = nc.exch2_txGlobalo - 1
j0 = nc.exch2_tyGlobalo - 1
ie = i0 + sNx
je = j0 + sNy
self._i0.append(i0)
self._j0.append(j0)
self._ie.append(ie)
self._je.append(je)
self._nx = max(self._nx, ie)
self._ny = max(self._ny, je)
# make ie, je relative to end (for non-tracer points)
for i in range(len(self._i0)):
ie = self._ie[i] - self._nx
je = self._je[i] - self._ny
self._ie[i] = ie or None
self._je[i] = je or None
elif layout == 'faces':
self._nx = {}
self._ny = {}
for nc in self.nc:
fn = nc.exch2_myFace
i0 = nc.exch2_tBasex
j0 = nc.exch2_tBasey
ie = i0 + sNx
je = j0 + sNy
self._fn.append(fn)
self._i0.append(i0)
self._j0.append(j0)
self._ie.append(ie)
self._je.append(je)
self._nx[fn] = max(self._nx.get(fn, 0), ie)
self._ny[fn] = max(self._ny.get(fn, 0), je)
# make ie, je relative to end (for non-tracer points)
for i in range(len(self._fn)):
fn = self._fn[i]
ie = self._ie[i] - self._nx[fn]
je = self._je[i] - self._ny[fn]
self._ie[i] = ie or None
self._je[i] = je or None
self._fns = sorted(self._nx.keys())
self._nf = len(self._fns)
for i in range(len(self._fn)):
self._fn[i] = self._fns.index(self._fn[i])
self._nx = np.array([self._nx[fn] for fn in self._fns])
self._ny = np.array([self._ny[fn] for fn in self._fns])
else:
raise ValueError('Unknown layout: {}'.format(layout))
# dimensions
self.dimensions = {}
for k, n in self.nc[0].dimensions.items():
# compute size of dimension in global array for X* and Y*
if k[0] == 'X':
n += self._nx - sNx
if k[0] == 'Y':
n += self._ny - sNy
self.dimensions[k] = n
# variables
var0 = self.nc[0].variables
# find size of record dimension first
if 'T' in self.dimensions and self.dimensions['T'] is None:
self.times = list(var0.get('T', [])[:])
self.iters = list(var0.get('iter', self.times)[:])
self.nrec = len(self.iters)
self.variables = dict((k, MNCVariable(self, k)) for k in var0)
def open_db(self, filepath, mode="r"):
"""Open the netcdf database file"""
if mode not in "rw":
raise ExodusIIFileError("{0}: bad read/write mode".format(mode))
return nc.netcdf_file(filepath, mode)
refnoitersort,trns,dimnoitersort = zip(*sorted(zip(refnoiter,range(len(refnoiter)),dimnoiter),key=itemgetter(0,1)))
icecube =rwicecube(fstream,shp,refiter,dimiter,dimiterpos,refnoitersort,dimnoitersort,None,vtype,vsize,voffset,rwchsize,'read')
# build the 'inverse permutation' operator for tranposition before writeout
inv = range(len(trns))
for itrns, etrns in enumerate(trns):
inv[etrns] = itrns
return np.transpose(icecube,inv)
fnin = '/home/hendrik/data/belgium_aq/rcm/aq09/stage1/int2lm/laf2009010100_urb_ahf.nc'
print fnin
# fobjin = open(fnin,'rb')
fin = netcdf.netcdf_file(fnin,'r')
fnout = '/home/hendrik/data/belgium_aq/rcm/aq09/stage1/int2lm/laf2009010100_urb_ahf2.nc'
os.system('rm '+fnout)
print fnout
# fobjout = open(fnout,'wb+')
fout = NetCDF.NetCDFFile(fnout,'w')
fnpointout = '/home/hendrik/data/belgium_aq/rcm/aq09/stage1/int2lm/laf2009010100_urb_ahf4.nc'
os.system('rm '+fnpointout)
print fnpointout
# fobjout = open(fnpointout,'wb+')
fpointout = open(fnpointout,'w')
# we kunnen eens proberen om een variabele aan te maken met een vooraf gespecifieerde dimensie!
datin = [[fin,'QV'],[fin,'rlat']]
def __init__(self, fpatt, layout=None, multitime=False):
fnames = glob.glob(fpatt)
# if multitime:
# iters = [ f[-18:-8] for f in fnames if f.endswith('.t001.nc') ]
# iters.sort()
# fnames_first = [ f for f in fnames if f[-18:-8] == iters[0] ]
# else:
# fnames_first = fnames
fnames.sort()
# open files
self.nc = [netcdf_file(f, "r") for f in fnames]
# global attributes
# get from first file, but remove/reset tile-specific ones
self._attributes = getattributes(self.nc[0], _exclude_global)
self._attributes["tile_number"] = 1
self._attributes["bi"] = 1
self._attributes["bj"] = 1
haveexch2 = False
for k in self._attributes.keys():
if k.startswith("exch2_"):
del self._attributes[k]
haveexch2 = True
sNx = self.sNx
sNy = self.sNy
ntx = self.nSx * self.nPx
nty = self.nSy * self.nPy
if layout is None:
if haveexch2:
layout = "exch2"
else:
layout = "model"
self.layout = layout
# precompute indices
self._i0 = []
self._ie = []
self._j0 = []
self._je = []
self._fn = []
self._nf = 0
if layout == "model":
self._nx = self.Nx
self._ny = self.Ny
for nc in self.nc:
tn = nc.tile_number
bj, bi = divmod(tn - 1, ntx)
ie = sNx * (bi + 1 - ntx)
je = sNy * (bj + 1 - nty)
self._i0.append(sNx * bi)
self._j0.append(sNy * bj)
self._ie.append(ie or None)
self._je.append(je or None)
elif layout == "exch2":
self._nx = 0
self._ny = 0
for nc in self.nc:
i0 = nc.exch2_txGlobalo - 1
j0 = nc.exch2_tyGlobalo - 1
ie = i0 + sNx
je = j0 + sNy
self._i0.append(i0)
self._j0.append(j0)
self._ie.append(ie)
self._je.append(je)
self._nx = max(self._nx, ie)
self._ny = max(self._ny, je)
# make ie, je relative to end (for non-tracer points)
for i in range(len(self._i0)):
ie = self._ie[i] - self._nx
je = self._je[i] - self._ny
self._ie[i] = ie or None
self._je[i] = je or None
elif layout == "faces":
self._nx = {}
self._ny = {}
for nc in self.nc:
fn = nc.exch2_myFace
i0 = nc.exch2_tBasex
j0 = nc.exch2_tBasey
ie = i0 + sNx
je = j0 + sNy
self._fn.append(fn)
self._i0.append(i0)
self._j0.append(j0)
self._ie.append(ie)
self._je.append(je)
self._nx[fn] = max(self._nx.get(fn, 0), ie)
self._ny[fn] = max(self._ny.get(fn, 0), je)
# make ie, je relative to end (for non-tracer points)
for i in range(len(self._fn)):
fn = self._fn[i]
ie = self._ie[i] - self._nx[fn]
je = self._je[i] - self._ny[fn]
self._ie[i] = ie or None
self._je[i] = je or None
self._fns = sorted(self._nx.keys())
self._nf = len(self._fns)
for i in range(len(self._fn)):
self._fn[i] = self._fns.index(self._fn[i])
self._nx = np.array([self._nx[fn] for fn in self._fns])
self._ny = np.array([self._ny[fn] for fn in self._fns])
else:
raise ValueError("Unknown layout: {}".format(layout))
# dimensions
self.dimensions = {}
for k, n in self.nc[0].dimensions.items():
# compute size of dimension in global array for X* and Y*
if k[0] == "X":
n += self._nx - sNx
if k[0] == "Y":
n += self._ny - sNy
self.dimensions[k] = n
# variables
var0 = self.nc[0].variables
# find size of record dimension first
if "T" in self.dimensions and self.dimensions["T"] is None:
self.times = list(var0.get("T", [])[:])
self.iters = list(var0.get("iter", self.times)[:])
self.nrec = len(self.iters)
self.variables = dict((k, MNCVariable(self, k)) for k in var0)
def open_db(self, filepath):
"""Open the netcdf database file"""
return nc.netcdf_file(filepath, "r")
datout = [[fout,'QV'],[fout,'TEST']]
# adtypeoutspec = [None,None] # to be obtained automatically from the data output stream (if it already exists)
# selection of function dimension input
func = lambda x, y: (np.array([[[np.mean(x)]],[[np.mean(x)]]],dtype=np.float) , np.array([[[np.mean(x)]],[[np.mean(x)]]],dtype=np.float)) # *(1.+np.zeros(x.shape))
dnamsel = ('rlon','time','t')
# obtain definitions of the variable stream input
vsdin = [] # input variable stream definitions
for idatin,edatin in enumerate(datin):
# read in scipy.netcdf mode to obtain varariable offsets
# obtain file name from open netcdf!! very nasty!!!
ncfn = str(datin[idatin][0])[19:(str(datin[idatin][0]).index("'",19))]
nctemp = netcdf.netcdf_file(ncfn,'r')
# nctemp = datin[idatin][0]
vsdin.append(dict())
vsdin[idatin]['dnams'] = []
for idim,edim in enumerate(nctemp.variables[datin[idatin][1]].dimensions):
vsdin[idatin]['dnams'].append(str(edim))
vsdin[idatin]['dims'] = list(nctemp.variables[datin[idatin][1]].shape)
vsdin[idatin]['itemsize'] = nctemp.variables[datin[idatin][1]].itemsize()
vsdin[idatin]['dtype'] = nctemp.variables[datin[idatin][1]]._dtype
vsdin[idatin]['voffset'] = nctemp.variables[datin[idatin][1]]._voffset
nctemp.close()
# obtain definitions of the variable stream output
vsdout = [] # input variable stream definitions
for idatout,edatout in enumerate(datout):
