def __init__(self, rf, multi=False):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = OpenRecordFile(rf)
self.__readheader()
irr_record_type = dtype(
dict(names=(
['SPAD', 'DATE', 'TIME', 'PAGRID', 'NEST', 'I', 'J', 'K'] +
['RXN_%02d' % i for i in range(1, self.nrxns + 1)] + ['EPAD']),
formats=(['>i', '>i', '>f', '>i', '>i', '>i', '>i', '>i'] +
['>f' for i in range(1, self.nrxns + 1)] + ['>i'])))
varkeys = [i for i in irr_record_type.names[8:-1]] + ['TFLAG']
self.groups = defaultdict(PseudoNetCDFFile)
padatatype = []
pavarkeys = []
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('VAR', len(varkeys) - 1)
self.createDimension('DATE-TIME', 2)
for di, domain in enumerate(self._padomains):
dk = 'PA%02d' % (di + 1)
prefix = dk + '_'
pavarkeys.extend([prefix + k for k in varkeys])
grp = self.groups[dk]
for propk, propv in domain.items():
setattr(grp, propk, propv)
grp.createDimension('TSTEP', self.time_step_count)
grp.createDimension('VAR', len(varkeys) - 1)
grp.createDimension('DATE-TIME', 2)
grp.createDimension('COL', domain['iend'] - domain['istart'] + 1)
grp.createDimension('ROW', domain['jend'] - domain['jstart'] + 1)
grp.createDimension('LAY', domain['tlay'] - domain['blay'] + 1)
padatatype.append(
(dk, irr_record_type, (len(grp.dimensions['ROW']),
len(grp.dimensions['COL']),
len(grp.dimensions['LAY']))))
if len(self._padomains) == 1:
self.createDimension('COL',
domain['iend'] - domain['istart'] + 1)
self.createDimension('ROW',
domain['jend'] - domain['jstart'] + 1)
self.createDimension('LAY',
domain['tlay'] - domain['blay'] + 1)
for propk, propv in domain.items():
setattr(grp, propk, propv)
exec(
"""def varget(k):
return self._irr__variables('%s', k)""" % dk, dict(self=self),
locals())
if len(self._padomains) == 1:
self.variables = PseudoNetCDFVariables(varget, varkeys)
else:
grp.variables = PseudoNetCDFVariables(varget, varkeys)
self.__memmaps = memmap(self.__rffile.infile.name, dtype(padatatype),
'r', self.data_start_byte)
def __init__(self, rf, rows=None, cols=None):
f = open(rf, 'rb')
f.seek(0, 2)
flen = f.tell()
offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8
self.__memmap = memmap(rf, dtype='>f', mode='r', offset=offset)
cldhdrlen = offset - 20
line1fmt = '>i%dciiii' % cldhdrlen
ncols, nrows, nlays = struct.unpack(line1fmt,
open(rf, 'rb').read(offset))[-4:-1]
self.createDimension('COL', ncols)
self.createDimension('ROW', nrows)
self.createDimension('LAY', nlays)
mydt = '>i4,S%d,>i4,>i4,>i4,>i4,>i4,>f4,>i4' % cldhdrlen
header = np.fromfile(rf, dtype=mydt, count=1)[0]
self.FILEDESC = ''.join(header[1].decode())
self.STIME, self.SDATE = header.tolist()[-2:]
if self.SDATE < 10000:
self.SDATE += 2000000
if (((ncols != cols and cols is not None)
or (rows != rows and rows is not None))):
warn(('Files says cols = %d, rows = %d, and lays = %d; ' +
'you said cols = %d and rows = %d') %
(ncols, nrows, nlays, cols, rows))
self.createDimension('DATE-TIME', 2)
datasize = (flen - offset)
# Try 5 first (contemporary)
# Try 3 second (old)
# end on 5 as a failsafe
for nvars in [5, 3, 5]:
timesize = (nvars * nlays * (nrows * ncols + 2) * 4 + 16)
if (datasize % timesize) == 0:
break
else:
warn('File appears incomplete using 3 (v4.2) or 5 ' +
'variables (>=v4.3); expected to fail')
if nvars < 5:
self.VERSION = '<4.3'
varkeys = ['CLOUD', 'PRECIP', 'COD', 'TFLAG']
else:
self.VERSION = '>=4.3'
varkeys = ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG']
ntimes = datasize // timesize
self.createDimension('TSTEP', ntimes)
self.createDimension('VAR', len(varkeys) - 1)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :]
def __init__(self, rf, rows=None, cols=None):
f = open(rf, 'rb')
f.seek(0, 2)
flen = f.tell()
offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8
self.__memmap = memmap(rf, dtype='>f', mode='r', offset=offset)
ncols, nrows, nlays = struct.unpack({
35: '>i15ciiii',
40: '>i20ciiii'
}[offset],
open(rf, 'rb').read(offset))[-4:-1]
self.createDimension('COL', ncols)
self.createDimension('ROW', nrows)
self.createDimension('LAY', nlays)
header = np.fromfile(rf,
dtype={
35: '>i4,S15,>i4,>i4,>i4,>i4,>i4,>f4,>i4',
40: '>i4,S20,>i4,>i4,>i4,>i4,>i4,>f4,>i4'
}[offset],
count=1)[0]
self.FILEDESC = ''.join(header[1].decode())
self.STIME, self.SDATE = header.tolist()[-2:]
if self.SDATE < 10000:
self.SDATE += 2000000
if (ncols != cols and cols != None) or (rows != rows and rows != None):
warn(
'Files says cols = %d, rows = %d, and lays = %d; you said cols = %d and rows = %d'
% (ncols, nrows, nlays, cols, rows))
self.createDimension('DATE-TIME', 2)
self.VERSION, varkeys = {
35: ('<4.3', ['CLOUD', 'PRECIP', 'COD', 'TFLAG']),
40: ('4.3', ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'])
}[offset]
self.createDimension('TSTEP',
(flen - offset) // ((len(varkeys) - 1) * nlays *
(nrows * ncols + 2) * 4 + 16))
self.createDimension('VAR', len(varkeys) - 1)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :]
def __init__(self, rf, rows=None, cols=None):
self.rffile = OpenRecordFile(rf)
self.id_size = struct.calcsize(self.id_fmt)
self.__readheader()
self.__gettimestep()
if rows is None and cols is None:
rows = self.cell_count
cols = 1
elif rows is None:
rows = self.cell_count / cols
elif cols is None:
cols = self.cell_count / rows
else:
if cols * rows != self.cell_count:
raise ValueError(
("The product of cols (%d) and rows (%d) " +
"must equal cells (%d)") % (cols, rows, self.cell_count))
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('COL', cols)
self.createDimension('ROW', rows)
self.createDimension('LAY', self.nlayers)
self.createDimension('SURF', 1)
self.variables = PseudoNetCDFVariables(self.__var_get,
['AIRTEMP', 'SURFTEMP'])
def __init__(self, rf, rows = None,cols = None):
f = open(rf, 'rb')
f.seek(0, 2)
flen = f.tell()
offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8
self.__memmap = memmap(rf, dtype = '>f', mode = 'r', offset = offset)
ncols, nrows, nlays = struct.unpack({35:'>i15ciiii', 40:'>i20ciiii'}[offset], open(rf, 'rb').read(offset))[-4:-1]
self.createDimension('COL', ncols)
self.createDimension('ROW', nrows)
self.createDimension('LAY', nlays)
header = np.fromfile(rf, dtype = {35:'>i4,S15,>i4,>i4,>i4,>i4,>i4,>f4,>i4', 40:'>i4,S20,>i4,>i4,>i4,>i4,>i4,>f4,>i4'}[offset], count = 1)[0]
self.FILEDESC = ''.join(header[1].decode())
self.STIME, self.SDATE = header.tolist()[-2:]
if self.SDATE < 10000:
self.SDATE+=2000000
if (ncols!=cols and cols!=None) or (rows!=rows and rows!=None):
warn('Files says cols = %d, rows = %d, and lays = %d; you said cols = %d and rows = %d' % (ncols, nrows, nlays, cols, rows))
self.createDimension('DATE-TIME', 2)
self.VERSION, varkeys = {35:('<4.3', ['CLOUD', 'PRECIP', 'COD', 'TFLAG']), 40:('4.3', ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'])}[offset]
self.createDimension('TSTEP', (flen - offset) // ((len(varkeys) - 1) * nlays * (nrows * ncols + 2) * 4 + 16))
self.createDimension('VAR', len(varkeys) - 1)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
self.SDATE,self.STIME = self.variables['TFLAG'][0, 0, :]
def __init__(self, rf, chemparam=None):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.rffile = OpenRecordFile(rf)
if chemparam is None:
self._aerosol_names = None
else:
self._aerosol_names = get_chemparam_names(chemparam)
self.padded_time_hdr_size = struct.calcsize(self.time_hdr_fmt + "ii")
self.__readheader()
self.__gettimestep()
self.dimensions = {}
self.createDimension('LAY', self.nlayers)
self.createDimension('COL', self.nx)
self.createDimension('ROW', self.ny)
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('DATE-TIME', 2)
self.variables = PseudoNetCDFVariables(
self.__var_get, [sn.strip() for sn in self.spcnames])
def __init__(self,rf,proc_dict=None,units='umol/m**3', oldnames = False, **props):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
Keywords (i.e., props) for projection: P_ALP, P_BET, P_GAM, XCENT, YCENT, XORIG, YORIG, XCELL, YCELL
"""
if proc_dict!=None:
self.proc_dict=proc_dict
else:
self.proc_dict=None
self.__rffile=open(rf, 'rb')
self.__rffile.seek(0,2)
if self.__rffile.tell()<2147483648:
warn("For greater speed on files <2GB use ipr_memmap")
self.__rffile.seek(0,0)
self.units=units
self.__readheader()
self.__setDomain__()
self.__gettimestep()
tdim = self.createDimension('TSTEP',self.NSTEPS)
tdim.setunlimited(True)
self.createDimension('DATE-TIME',2)
self.createDimension('VAR',self.NSPCS*self.NPROCESS)
varkeys=["_".join([j[1],j[0]]) for j in [i for i in cartesian([i.strip() for i in self.spcnames['SPECIES'].tolist()],self.proc_dict.keys())]]+['TFLAG']
self.variables=PseudoNetCDFVariables(self.__variables,varkeys)
for k, v in props.items():
setattr(self, k, v)
add_cf_from_ioapi(self)
def __init__(self, rf, rows, cols):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.rffile = OpenRecordFile(rf)
self.id_size = struct.calcsize(self.id_fmt)
self.__readheader()
self.__gettimestep()
if rows == None and cols == None:
rows = self.cell_count
cols = 1
elif rows == None:
rows = self.cell_count / cols
elif cols == None:
cols = self.cell_count / rows
else:
if cols * rows != self.cell_count:
raise ValueError(
"The product of cols (%d) and rows (%d) must equal cells (%d)"
% (cols, rows, self.cell_count))
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('ROW', rows)
self.createDimension('COL', cols)
self.createDimension('LAY', self.nlayers)
self.variables = PseudoNetCDFVariables(self.__var_get,
['HGHT', 'PRES'])
def __init__(* args, **kwds):
self = args[0]
self.__child = self.__childclass__(* args[1:], **kwds)
self.dimensions = self.__child.dimensions
self.variables = PseudoNetCDFVariables(
self.__variables__,
self.__child.variables.keys() + self.__addvars__)
def __init__(self, ncffile, dimension, oldres, newres,
repeat_method=repeat, condense_method=sum, nthick=0):
from PseudoNetCDF.sci_var import Pseudo2NetCDF
PseudoNetCDFFile.__init__(self)
self.__dimension = array(dimension, ndmin=1)
oldres = array(oldres, ndmin=1)
newres = array(newres, ndmin=1)
self.__mesh = newres / oldres.astype('f')
self.__condense = condense_method
self.__repeat = repeat_method
self.__file = ncffile
self.__nthick = nthick
if not logical_or((self.__mesh % 1) == 0,
(1. / self.__mesh) % 1 == 0).any():
raise ValueError("One resolution must be a factor of the other.")
Pseudo2NetCDF().addDimensions(self.__file, self)
any_non_time_key = [
k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
for dk, dfactor in zip(self.__dimension, 1. / self.__mesh):
dimo = self.dimensions[dk]
ndimo = self.createDimension(str(dk), len(dimo) * dfactor)
ndimo.setunlimited(dimo.isunlimited())
v = self.__file.variables[any_non_time_key]
v = self.__method(v)
self.variables = PseudoNetCDFVariables(
self.__variables, self.__file.variables.keys())
def __init__(self, rf, mode='r'):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = rf
# Establish dimensions
self.createDimension('DATE-TIME', 2)
self.__readheader(mode)
# Add IOAPI metavariables
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.NVARS = len(self.dimensions['VAR'])
self.NSTEPS = len(self.dimensions['TSTEP'])
varlist = "".join([i.ljust(16) for i in self.__var_names__])
setattr(self, 'VAR-LIST', varlist)
self.GDTYP = 2
# Create variables
self.variables = PseudoNetCDFVariables(self.__variables,
self.__var_names__ + ['TFLAG'])
# Initialize time maps
date = self.__memmap__['DATE']['DATE']
time = self.__memmap__['DATE']['TIME']
self.variables['TFLAG'] = ConvertCAMxTime(date, time, self.NVARS)
self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :]
def __init__(self,
rf,
proc_dict=None,
units='umol/m**3',
oldnames=False,
**props):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
Keywords (i.e., props) for projection: P_ALP, P_BET, P_GAM, XCENT, YCENT, XORIG, YORIG, XCELL, YCELL
"""
if proc_dict != None:
self.proc_dict = proc_dict
else:
self.proc_dict = None
self.__rffile = open(rf, 'rb')
self.__rffile.seek(0, 2)
if self.__rffile.tell() < 2147483648:
warn("For greater speed on files <2GB use ipr_memmap")
self.__rffile.seek(0, 0)
self.units = units
self.__readheader()
self.__setDomain__()
self.__gettimestep()
tdim = self.createDimension('TSTEP', self.NSTEPS)
tdim.setunlimited(True)
self.createDimension('DATE-TIME', 2)
self.createDimension('VAR', self.NSPCS * self.NPROCESS)
varkeys = [
"_".join([j[1], j[0]]) for j in [
i for i in cartesian([
i.strip()
for i in char.decode(self.spcnames['SPECIES']).tolist()
], self.proc_dict.keys())
]
] + ['TFLAG']
self.variables = PseudoNetCDFVariables(self.__variables, varkeys)
for k, v in props.items():
setattr(self, k, v)
try:
add_cf_from_ioapi(self)
except:
pass
def __init__(self, ncffile, tslice = slice(None), kslice = slice(None), jslice = slice(None), islice = slice(None)):
self.__file = ncffile
self.__idx = [tslice, kslice, jslice, islice]
self.dimensions = self.__file.dimensions.copy()
any_non_time_key = [k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
self.dimensions['TSTEP'], self.dimensions['LAY'], self.dimensions['ROW'], self.dimensions['COL'] \
= map(lambda x: PseudoNetCDFDimension(None, None, x), self.__file.variables[any_non_time_key][self.__idx].shape)
self.variables = PseudoNetCDFVariables(self.__variables, self.__file.variables.keys())
def __init__(self,path):
self.__file=open(path,'r')
self.__readheader()
self.__setglobalprops()
self.__setprivateprops()
self.__setdimensions()
self.__settimeprops()
self.variables=PseudoNetCDFVariables(self.__readonespc,self.__spc_names)
def __init__(self, rf, rows=None, cols=None):
"""
Initialization included reading the header and learning
about the format.
rows - number of rows in domain (defaults: 1)
cols - number of columns in domain (defaults: size)
"""
self.rffile = rf
self.__memmap = memmap(self.rffile, '>f', 'r', offset=0)
if rows is None and cols is None:
rows = 1
cols = self.__memmap[[-1]].view('>i')[0] // 4 - 2
self.__record_items = rows * cols + 4
self.__records = self.__memmap.shape[0] // self.__record_items
time_date = array(
self.__memmap.reshape(self.__records, self.__record_items)[:, 1:3])
lays = where(time_date != time_date[newaxis, 0])[0][0]
new_hour = slice(0, None, lays)
dates = time_date[:, 1].view('>i')
times = time_date[:, 0]
self.__tflag = array([dates[new_hour], times[new_hour]],
dtype='>f').swapaxes(0, 1)
time_steps = self.__records / lays
self.createDimension('VAR', 1)
self.createDimension('TSTEP', time_steps)
self.createDimension('COL', cols)
self.createDimension('ROW', rows)
self.createDimension('LAY', lays)
self.createDimension('DATE-TIME', 2)
self.FTYPE = 1
self.NVARS = 1
self.NCOLS = cols
self.NROWS = rows
self.NLAYS = lays
self.NTHIK = 1
self.variables = PseudoNetCDFVariables(self.__variables,
[self.var_name])
v = self.variables['TFLAG'] = ConvertCAMxTime(self.__tflag[:, 0],
self.__tflag[:, 1], 1)
self.SDATE, self.STIME = v[0, 0, :]
def __init__(self, rf, endian='big', mode='r', **kwds):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = rf
ep = self.__endianprefix = dict(big='>', little='<')[endian]
self.variables = {}
for key, val in kwds.items():
setattr(self, key, val)
self.__memmap = memmap(self.__rffile, ep + 'f', mode)
self.__emiss_hdr_fmt = dtype(dict(
names=['SPAD', 'name', 'note', 'itzon', 'nspec', 'ibdate',
'btime', 'iedate', 'etime', 'EPAD'],
formats=['i', '(10,4)S1', '(60,4)S1', 'i', 'i', 'i', 'f', 'i',
'f', 'i'])).newbyteorder(ep)
self.__grid_hdr_fmt = dtype(dict(
names=['SPAD', 'plon', 'plat', 'iutm', 'xorg', 'yorg', 'delx',
'dely', 'nx', 'ny', 'nz', 'iproj', 'istag', 'tlat1',
'tlat2', 'rdum5', 'EPAD'],
formats=['i', 'f', 'f', 'i', 'f', 'f', 'f', 'f', 'i', 'i', 'i',
'i', 'i', 'f', 'f', 'f', 'i'])).newbyteorder(ep)
self.__cell_hdr_fmt = dtype(dict(
names=['SPAD', 'ione1', 'ione2', 'nx', 'ny', 'EPAD'],
formats=['i', 'i', 'i', 'i', 'i', 'i'])).newbyteorder(ep)
self.__spc_hdr_fmt = dtype('(10,4)' + 'S1').newbyteorder(ep)
self.__time_hdr_fmt = dtype(dict(
names=['SPAD', 'ibdate', 'btime', 'iedate', 'etime', 'EPAD'],
formats=['i', 'i', 'f', 'i', 'f', 'i'])).newbyteorder(ep)
self.__nstk_hdr_fmt = dtype(dict(
names=['SPAD', 'ione', 'nstk', 'EPAD'],
formats=['i', 'i', 'i', 'i'])).newbyteorder(ep)
self.__stk_prop_fmt = dtype(dict(
names=['XSTK', 'YSTK', 'HSTK', 'DSTK', 'TSTK', 'VSTK'],
formats=['f', 'f', 'f', 'f', 'f', 'f'])).newbyteorder(ep)
self.__stk_time_prop_fmt = dtype(dict(
names=['IONE', 'ITWO', 'KCELL', 'FLOW', 'PLMHT'],
formats=['i', 'i', 'f', 'f', 'f'])).newbyteorder(ep)
self.__globalheader()
varkeys = ['ETFLAG', 'TFLAG', 'XSTK', 'YSTK', 'HSTK', 'DSTK', 'TSTK',
'VSTK', 'IONE', 'ITWO', 'KCELL', 'FLOW', 'PLMHT',
'NSTKS'] + [i.strip() for i in self.__spc_names]
self.SPC_NAMES = ''.join([s.ljust(16) for s in self.__spc_names])
setattr(self, 'VAR-LIST', "".join([i.ljust(16) for i in varkeys]))
self.variables = PseudoNetCDFVariables(self.__variables, varkeys)
self.__time_stks()
self.createDimension('NSTK', self.__nstk_hdr['nstk'])
def __init__(self, ncffile, tslice=slice(None), kslice=slice(None),
jslice=slice(None), islice=slice(None)):
self.__file = ncffile
self.__idx = [tslice, kslice, jslice, islice]
self.dimensions = self.__file.dimensions.copy()
any_non_time_key = [
k for k in self.__file.variables.keys() if 'TFLAG' not in k][0]
tmpvar = self.__file.variables[any_non_time_key][self.__idx]
ntimes, nlays, nrows, ncols = tmpvar.shape
self.dimensions['TSTEP'] = PseudoNetCDFDimension(None, None, ntimes)
self.dimensions['LAY'] = PseudoNetCDFDimension(None, None, nlays)
self.dimensions['ROW'] = PseudoNetCDFDimension(None, None, nrows)
self.dimensions['COL'] = PseudoNetCDFDimension(None, None, ncols)
self.variables = PseudoNetCDFVariables(
self.__variables, self.__file.variables.keys())
def __init__(self, files):
self.__files = files
keys = []
for f in self.__files:
for k, d in f.dimensions.items():
if len(d)==1 and k=='LAY':
k = 'SURFLAY'
if k not in self.dimensions.keys():
self.createDimension(k, len(d))
keys.extend(f.variables.keys())
for k in f.__dict__.keys():
if k not in self.__dict__.keys():
setattr(self, k, getattr(f, k))
keys = list(set(keys))
self.variables = PseudoNetCDFVariables(self.__variables, keys)
self.createDimension('VAR', len(keys) - 1)
def __init__(self, rf, mode='r'):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = rf
self.__mode = mode
self.createDimension('DATE-TIME', 2)
self.__readheader()
# Add IOAPI metavariables
nlays = self.NLAYS = len(self.dimensions['LAY'])
nrows = self.NROWS = len(self.dimensions['ROW'])
ncols = self.NCOLS = len(self.dimensions['COL'])
nvars = self.NVARS = len(self.dimensions['VAR'])
nsteps = self.NSTEPS = len(self.dimensions['TSTEP'])
setattr(
self, 'VAR-LIST',
"".join([i.ljust(16)
for i in self.__var_names__] + ['TFLAG'.ljust(16)]))
self.GDTYP = 2
name = self.__emiss_hdr['name'][0, :, 0].copy().view('S10')[0]
note = self.__emiss_hdr['note'][0, :, 0].copy().view('S60')[0]
if hasattr(name, 'decode'):
name = name.decode()
if hasattr(note, 'decode'):
note = note.decode()
self.NAME = name
self.NOTE = note
self.ITZON = self.__emiss_hdr['itzon'][0]
# Create variables
self.variables = PseudoNetCDFVariables(
self.__variables, self.__var_names__ + ['TFLAG', 'ETFLAG'])
self.variables['TFLAG'] = ConvertCAMxTime(
self.__memmap__['DATE']['BDATE'], self.__memmap__['DATE']['BTIME'],
self.NVARS)
self.variables['ETFLAG'] = ConvertCAMxTime(
self.__memmap__['DATE']['EDATE'], self.__memmap__['DATE']['BTIME'],
self.NVARS)
self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :]
def __init__(self, rffile, rows, cols, outunit='m/s', endhour=True,
forcestaggered=False):
self.__windfile = reg_wind(rffile, rows, cols)
self.createDimension('TSTEP', self.__windfile.dimensions['TSTEP'] - 1)
self.createDimension('LAY', self.__windfile.dimensions['LAY'])
self.createDimension('ROW', self.__windfile.dimensions['ROW'])
self.createDimension('COL', self.__windfile.dimensions['COL'])
self.createDimension('VAR', self.__windfile.dimensions['VAR'])
self.createDimension(
'DATE-TIME', self.__windfile.dimensions.get('DATE-TIME', 2))
self.__outunit = outunit
self.__force_stagger = forcestaggered
self.variables = PseudoNetCDFVariables(
self.__variables, ['V', 'U', 'TFLAG'])
self.__timeslice = {True: slice(
1, None), False: slice(None, -1)}[endhour]
def __init__(self, rffile, rows, cols):
rf = OpenRecordFile(rffile)
self.__time_hdr_fmts = {12: "fii", 8: "fi"}[rf.record_size]
self.__time_hdr_fmts_size = rf.record_size
self.STIME, self.SDATE = rf.unpack("fi")
rf.next()
lays = 1
record_size = rf.record_size
while rf.record_size == record_size:
lays += 1
rf.next()
self.__dummy_length = (rf.record_size + 8) / 4
lays //= 2
record = rows * cols * 4 + 8
total_size = self.__dummy_length
times = 0
while total_size < rf.length:
times += 1
total_size += record * 2 * lays + self.__time_hdr_fmts_size + 8
times -= 1
self.variables = OrderedDict
del rf
self.createDimension('TSTEP', times)
self.createDimension('DATE-TIME', 2)
self.createDimension('LAY', lays)
self.createDimension('ROW', rows)
self.createDimension('COL', cols)
self.createDimension('VAR', 2)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.__memmap = memmap(rffile, '>f', 'r', offset=0)
if self.__time_hdr_fmts_size == 12:
self.LSTAGGER = self.__memmap[3].view('i')
else:
self.LSTAGGER = nan
self.variables = PseudoNetCDFVariables(self.__variables,
['TFLAG', 'U', 'V'])
def __init__(self, rffile, rows, cols, outunit = {}, endhour = True):
self.__file = self.__reader__(rffile, rows, cols)
self.createDimension('TSTEP', len(self.__file.dimensions['TSTEP'] - 1))
self.createDimension('LAY', len(self.__file.dimensions['LAY']))
self.createDimension('ROW', len(self.__file.dimensions['ROW']))
self.createDimension('COL', len(self.__file.dimensions['COL']))
self.createDimension('VAR', len(self.__file.dimensions['VAR']))
if self.__file.dimensions.has('DATE-TIME'):
self.createDimension('DATE-TIME', len(self.__file.dimensions['DATE-TIME']))
else:
self.createDimension('DATE-TIME', 2)
self.__outunit = outunit
self.variables = PseudoNetCDFVariables(self.__variables, self.__file.variables.keys())
self.__timeslice = {True:slice(1, None), False:slice(None, -1)}[endhour]
v = self.createVariable('TFLAG', 'i', ('TSTEP', 'VAR', 'DATE-TIME'), keep = True)
v[:] = self.__file.variables['TFLAG'][self.__timeslice]
v.long_name = 'Time flag'
v.units = 'DATE-TIME'
def __init__(self, rf):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.rffile = OpenRecordFile(rf)
self.padded_time_hdr_size = struct.calcsize(self.time_hdr_fmt + "ii")
self.__readheader()
self.__gettimestep()
self.__gettimeprops()
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('STK', self.nstk)
varkeys = [
'XSTK', 'YSTK', 'HSTK', 'DSTK', 'TSTK', 'VSTK', 'KCELL', 'FLOW',
'PLMHT'
] + [i.strip() for i in self.spcnames]
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
def __init__(self, rf, rows=None, cols=None):
self.__memmap = memmap(rf, '>f', 'r', offset=0)
rowsXcols = self.__memmap[0].view('i') / 4 - 2
record_length = rowsXcols + 4
records = self.__memmap.size / record_length
times = self.__memmap.reshape(records, record_length)[:, 1:3]
self.STIME, self.SDATE = times[0]
for i, (t, d) in enumerate(times):
if (t, d) != (self.STIME, self.SDATE):
break
self.SDATE = self.SDATE.view('i')
self.createDimension('LAY', i / 2)
self.createDimension('TSTEP', times.shape[0] / i)
if rows == None and cols == None:
rows = rowsXcols
cols = 1
elif rows == None:
rows = rowsXcols / cols
elif cols == None:
cols = rowsXcols / rows
else:
if cols * rows != rowsXcols:
raise ValueError(
"The product of cols (%d) and rows (%d) must equal cells (%d)"
% (cols, rows, rowsXcols))
self.createDimension('ROW', rows)
self.createDimension('COL', cols)
self.createDimension('DATE-TIME', 2)
self.createDimension('VAR', 2)
self.NROWS = rows
self.NCOLS = cols
self.NLAYS = len(self.dimensions['LAY'])
self.NVARS = 2
self.NTHIK = 1
setattr(self, 'VAR-LIST', 'HGHT'.ljust(16) + 'PRES'.ljust(16))
self.variables = PseudoNetCDFVariables(self.__var_get,
['HGHT', 'PRES', 'TFLAG'])
def __init__(self, rf, rows=None, cols=None):
self.__memmap = memmap(rf, '>f', 'r', offset=0)
rowsXcols = self.__memmap[0].view('i') // 4 - 2
record_length = rowsXcols + 4
records = self.__memmap.size // record_length
times = self.__memmap.reshape(records, record_length)[:, 1:3]
self.STIME, self.SDATE = times[0]
for i, (t, d) in enumerate(times):
if (t, d) != (self.STIME, self.SDATE):
break
self.SDATE = self.SDATE.view('i')
self.createDimension('LAY', i - 1)
self.createDimension('TSTEP', times.shape[0] / i)
if rows is None and cols is None:
rows = rowsXcols
cols = 1
elif rows is None:
rows = rowsXcols / cols
elif cols is None:
cols = rowsXcols / rows
else:
if cols * rows != rowsXcols:
raise ValueError(
("The product of cols (%d) and rows (%d) " +
"must equal cells (%d)") % (cols, rows, rowsXcols))
self.createDimension('ROW', rows)
self.createDimension('COL', cols)
self.createDimension('DATE-TIME', 2)
self.createDimension('VAR', 2)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.variables = PseudoNetCDFVariables(
self.__var_get, ['AIRTEMP', 'SURFTEMP', 'TFLAG'])
def __init__(self, pig_path, mode = 'r'):
file_size = os.path.getsize(pig_path)
tmpmap = memmap(pig_path, dtype = '>i', shape = 7)
self.SDATE = tmpmap[1]
self.STIME = tmpmap[2:3].view('>f')[0]
self.NPIG = tmpmap[3]
record_length = int(tmpmap[-1]/4)
per_pig = record_length / self.NPIG
pig_dtype = dtype(
dict(
names = 'SPAD1 IDATE TIME NPIG IDUM EPAD1'.split()+['SPAD2']+['P%d_%d' % (i / per_pig,i % per_pig) for i in range(record_length)]+['EPAD2'],
formats = '>i >i >f >i >i >i >i'.split() + ['>f'] * record_length + ['>i']
)
)
items = float(file_size) / float(pig_dtype.itemsize)
assert(items == int(items))
self.NSTEPS = items = int(items)
self.__memmap__ = memmap(pig_path, shape = items, dtype = pig_dtype)
self.dimensions = dict(TSTEP = items, NPIG = self.NPIG)
self.variables = PseudoNetCDFVariables(self.__variables, pig_dtype.fields.keys())
def __init__(self, rf):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = rf
self.variables = {}
self.__memmap = memmap(self.__rffile, '>f', 'r')
self.__globalheader()
varkeys = [
'ETFLAG', 'TFLAG', 'XSTK', 'YSTK', 'HSTK', 'DSTK', 'TSTK', 'VSTK',
'IONE', 'ITWO', 'KCELL', 'FLOW', 'PLMHT', 'NSTKS'
] + [i.strip() for i in self.__spc_names]
self.SPC_NAMES = ''.join([s.ljust(16) for s in self.__spc_names])
setattr(self, 'VAR-LIST', "".join([i.ljust(16) for i in varkeys]))
self.variables = PseudoNetCDFVariables(self.__variables, varkeys)
self.__time_stks()
self.createDimension('NSTK', self.__nstk_hdr['nstk'])
def __init__(self, rf, units='ppm/hr', conva=None, nvarcache=None):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
rffile = self._rffile = open(rf)
rffile.seek(0, 2)
if rffile.tell() < 2147483648:
warn("For greater speed on files <2GB use ipr_memmap")
rffile.seek(0, 0)
self.__readheader()
self.__gettimestep()
self.units = units
#__conv is a conversion array that comes from ipr
#if units is not ppm/hr, conv must be provided
self.__conv = conva
if self.units != 'ppm/hr' and self.__conv == None:
raise ValueError(
"When units are provided, a conversion array dim(t,z,x,y) must also be provided"
)
varkeys = ['RXN_%02d' % i for i in range(1, self.NRXNS + 1)]
domain = self.padomains[0]
tdim = self.createDimension('TSTEP', int(self.NSTEPS))
tdim.setunlimited(True)
self.createDimension('COL', int(domain['iend'] - domain['istart'] + 1))
self.createDimension('ROW', int(domain['jend'] - domain['jstart'] + 1))
self.createDimension('LAY', int(domain['tlay'] - domain['blay'] + 1))
self.createDimension('DATE-TIME', 2)
self.createDimension('VAR', int(self.NRXNS))
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
self._nvarcache = nvarcache or int(ceil(self.NRXNS / 4.))
tflag = self.createVariable('TFLAG', 'i',
('TSTEP', 'VAR', 'DATE-TIME'))
tflag.units = '<YYYYJJJ, HHMMSS>'
tflag.var_desc = tflag.long_name = 'TFLAG'.ljust(16)
tflag[:] = array(self.timerange(),
dtype='i').reshape(self.NSTEPS, 1, 2)
def __init__(self, rf):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.rffile = OpenRecordFile(rf)
self.padded_time_hdr_size = struct.calcsize(self.time_hdr_fmt + "ii")
self.__readheader()
self.__gettimestep()
self.dimensions = {}
self.createDimension('LAY', self.nlayers)
self.createDimension('COL', self.nx)
self.createDimension('ROW', self.ny)
self.createDimension('TSTEP', self.time_step_count)
self.createDimension('DATE-TIME', 2)
self.variables = PseudoNetCDFVariables(self.__var_get,
map(str.strip, self.spcnames))
def __init__(self, rffile, sources={}, regions={}):
self.__child = uamiv(rffile)
# Use Use all species based keys
self.__sourcesbyNm = dict([(k[3:6], k[3:6])
for k in self.__child.variables.keys()
if k[:3] in ('NOX', 'VOC', 'O3V', 'O3N')])
self.__regionsbyNm = dict([(k[6:], k[6:])
for k in self.__child.variables.keys()
if k[:3] in ('NOX', 'VOC', 'O3V', 'O3N')])
# Create global keys
self.__sourcesbyNm[''] = tuple(self.__sourcesbyNm.keys())
self.__regionsbyNm[''] = tuple(self.__regionsbyNm.keys())
for k, v in sources.items():
if type(v) == str:
sources[k] = (v, )
for k, v in regions.items():
if type(v) == str:
regions[k] = (v, )
# Update with user supplied keys
self.__sourcesbyNm.update(sources)
self.__regionsbyNm.update(regions)
self.dimensions = self.__child.dimensions.copy()
spc_keys = ['NOX', 'VOC', 'O3N', 'O3V']
time_keys = ['I', 'D']
time_dim_keys = [(k[:1], k[1:6], k[6:]) for k in time_keys]
allkeys = [i for i in self.__child.variables.keys()]
for skey in spc_keys:
for src in sources.keys() + ['']:
for reg in regions.keys() + ['']:
allkeys.append(self.__delim.join([skey, src, reg]))
allkeys = [i for i in set(allkeys)]
self.variables = PseudoNetCDFVariables(self.__variables, allkeys)
def __init__(self,
path,
nogroup=False,
noscale=False,
vertgrid='GEOS-5-REDUCED'):
self.noscale = noscale
self._nogroup = nogroup
from ._vertcoord import geos_hyai, geos_hybi
self.vertgrid = vertgrid
# Ap [hPa]
self.Ap = geos_hyai[self.vertgrid]
# Bp [unitless]
self.Bp = geos_hybi[self.vertgrid]
self._gettracerinfo(path)
self._getdiaginfo(path)
self._data = data = np.memmap(path, mode='r', dtype='uint8')
header = data[:136].copy().view(_general_header_type)
self.ftype = header[0][1]
self.toptitle = header[0][4]
offset = 136
outpos = self._outpos = OrderedDict()
while offset < data.size:
tmp_hdr = data[offset:offset +
_hdr_size].view(_datablock_header_type)
cat = tmp_hdr['category'][0].strip()
if hasattr(cat, 'decode'):
cat = cat.decode()
trac = str(tmp_hdr['tracerid'][0])
if hasattr(trac, 'decode'):
trac = trac.decode()
key = cat + '_' + trac
tau0, tau1 = tmp_hdr['tau0'][0], tmp_hdr['tau1'][0]
dim = tuple(tmp_hdr['dim'][0].tolist())
skip = tmp_hdr['skip'][0]
outpos.setdefault(
key,
OrderedDict())[(tau0,
tau1)] = offset, offset + _hdr_size + skip, dim
offset += skip + _hdr_size
modelname, modelres, halfpolar, center180 = tmp_hdr[0].tolist()[1:5]
self.modelname = modelname
self.modelres = modelres
self.halfpolar = halfpolar
self.center180 = center180
tmpvariables = self._gcvars = OrderedDict()
for key in outpos:
tmpvar = gcvar(key, self)
if nogroup is True:
tmpkey = str(tmpvar.shortname)
elif nogroup is False:
tmpkey = tmpvar.category + str('_') + tmpvar.shortname
elif tmpvar.category in nogroup:
tmpkey = str(tmpvar.shortname)
else:
tmpkey = tmpvar.category + str('_') + tmpvar.shortname
tmpvariables[tmpkey] = tmpvar
self.modelres = tmpvar._header['modelres'][0]
self.halfpolar = tmpvar._header['halfpolar'][0]
self.center180 = tmpvar._header['center180'][0]
ntimes = [tmpvar.shape[0] for tmpkey, tmpvar in tmpvariables.items()]
levels = [tmpvar.shape[1] for tmpkey, tmpvar in tmpvariables.items()]
latitudes = [
tmpvar.shape[2] for tmpkey, tmpvar in tmpvariables.items()
]
longitudes = [
tmpvar.shape[3] for tmpkey, tmpvar in tmpvariables.items()
]
self._maxntimes = maxntimes = max(ntimes)
self.createDimension('time', maxntimes)
for time in set(ntimes):
if time != maxntimes:
self.createDimension('time%d' % time, time)
self.createDimension('layer', min(max(levels), self.Ap.size - 1))
for layer in set(levels):
self.createDimension('layer%d' % layer, layer)
self.createDimension('latitude', max(latitudes))
self.createDimension('longitude', max(longitudes))
self.createDimension('nv', 2)
self.variables = PseudoNetCDFVariables(self._addvar,
list(tmpvariables))
key = list(tmpvariables)[0]
tmpvar = self.variables[key]
add_lat(self, 'latitude', tmpvar)
add_lat(self, 'latitude_bounds', tmpvar)
add_lon(self, 'longitude', tmpvar)
add_lon(self, 'longitude_bounds', tmpvar)
add_vert(self, 'hyam')
add_vert(self, 'hyai')
add_vert(self, 'hybm')
add_vert(self, 'hybi')
add_vert(self, 'etai_pressure')
add_vert(self, 'etam_pressure')
for k, v in [('tau0', tmpvar._tau0), ('time', tmpvar._tau0),
('tau1', tmpvar._tau1)]:
tvar = self.createVariable(k, 'i', ('time', ))
tvar.units = 'hours since 1985-01-01 00:00:00 UTC'
tvar[:] = v[:]
def __init__(self,
rf,
mode='r',
P_ALP=None,
P_BET=None,
P_GAM=None,
XCENT=None,
YCENT=None,
GDTYP=None):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
"""
self.__rffile = rf
self.__mode = mode
self.createDimension('DATE-TIME', 2)
self.__readheader()
# Add IOAPI metavariables
nlays = self.NLAYS = len(self.dimensions['LAY'])
nrows = self.NROWS = len(self.dimensions['ROW'])
ncols = self.NCOLS = len(self.dimensions['COL'])
nvars = self.NVARS = len(self.dimensions['VAR'])
nsteps = self.NSTEPS = len(self.dimensions['TSTEP'])
setattr(self, 'VAR-LIST',
"".join([i.ljust(16) for i in self.__var_names__]))
self.NAME = self.__emiss_hdr['name'][0, :,
0].copy().view('S10')[0].decode()
self.NOTE = self.__emiss_hdr['note'][0, :,
0].copy().view('S60')[0].decode()
self.ITZON = self.__emiss_hdr['itzon'][0]
self.FTYPE = 1
self.VGTYP = 2
self.VGTOP = 10000.
self.VGLVLS = linspace(0, 1, self.NLAYS + 1)[::-1]
self.GDNAM = "CAMx "
self.UPNAM = "CAMx "
self.FILEDESC = "CAMx "
# Create variables
self.variables = PseudoNetCDFVariables(
self.__variables, self.__var_names__ + ['TFLAG', 'ETFLAG'])
tflag = ConvertCAMxTime(self.__memmap__['DATE']['BDATE'],
self.__memmap__['DATE']['BTIME'], self.NVARS)
etflag = ConvertCAMxTime(self.__memmap__['DATE']['EDATE'],
self.__memmap__['DATE']['ETIME'], self.NVARS)
tflagv = self.createVariable('TFLAG',
'i', ('TSTEP', 'VAR', 'DATE-TIME'),
values=tflag,
units='DATE-TIME',
long_name='TFLAG'.ljust(16),
var_desc='TFLAG'.ljust(80))
etflagv = self.createVariable('ETFLAG',
'i', ('TSTEP', 'VAR', 'DATE-TIME'),
values=etflag,
units='DATE-TIME',
long_name='ETFLAG'.ljust(16),
var_desc='Ending TFLAG'.ljust(80))
self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :]
self.TSTEP = self.variables['ETFLAG'][
0, 0, 1] - self.variables['TFLAG'][0, 0, 1]
if not P_ALP is None:
self.P_ALP = P_ALP
if not P_BET is None:
self.P_BET = P_BET
if not P_GAM is None:
self.P_GAM = P_GAM
if not XCENT is None:
self.XCENT = XCENT
if not YCENT is None:
self.YCENT = YCENT
if not GDTYP is None:
self.GDTYP = GDTYP
try:
add_cf_from_ioapi(self)
except Exception as e:
warn(repr(e))
pass
class ipr(PseudoNetCDFFile):
"""
ipr provides a PseudoNetCDF interface for CAMx
ipr files. Where possible, the inteface follows
IOAPI conventions (see www.baronams.com).
ex:
>>> ipr_path = 'camx_ipr.bin'
>>> rows,cols = 65,83
>>> iprfile = ipr(ipr_path,rows,cols)
>>> iprfile.variables.keys()
['TFLAG', 'SPAD_O3', 'DATE_O3', 'TIME_O3', 'SPC_O3',
'PAGRID_O3', 'NEST_O3', 'I_O3', 'J_O3', 'K_O3',
'INIT_O3', 'CHEM_O3', 'EMIS_O3', 'PTEMIS_O3',
'PIG_O3', 'WADV_O3', 'EADV_O3', 'SADV_O3', 'NADV_O3',
'BADV_O3', 'TADV_O3', 'DIL_O3', 'WDIF_O3', 'EDIF_O3',
'SDIF_O3', 'NDIF_O3', 'BDIF_O3', 'TDIF_O3', 'DDEP_O3',
'WDEP_O3', 'INORGACHEM_O3', 'ORGACHEM_O3', 'AQACHEM_O3',
'FCONC_O3', 'UCNV_O3', 'AVOL_O3', 'EPAD_O3']
>>> v = iprfile.variables['CHEM_O3']
>>> tflag = iprfile.variables['TFLAG']
>>> tflag.dimensions
('TSTEP', 'VAR', 'DATE-TIME')
>>> tflag[0,0,:]
array([2005185, 0])
>>> tflag[-1,0,:]
array([2005185, 240000])
>>> v.dimensions
('TSTEP', 'LAY', 'ROW', 'COL')
>>> v.shape
(25, 28, 65, 83)
>>> iprfile.dimensions
{'TSTEP': 25, 'LAY': 28, 'ROW': 65, 'COL': 83}
"""
__ipr_record_type ={
24: dtype(
dict(
names=['SPAD', 'DATE', 'TIME', 'SPC', 'PAGRID', 'NEST', 'I', 'J', 'K', 'INIT', 'CHEM', 'EMIS', 'PTEMIS', 'PIG', 'WADV', 'EADV', 'SADV', 'NADV', 'BADV', 'TADV', 'DIL', 'WDIF', 'EDIF', 'SDIF', 'NDIF', 'BDIF', 'TDIF', 'DDEP', 'WDEP', 'AERCHEM', 'FCONC', 'UCNV', 'AVOL', 'EPAD'],
formats=['>i', '>i', '>f', '>S10', '>i', '>i', '>i', '>i', '>i', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>i']
)
),
26: dtype(
dict(
names=['SPAD', 'DATE', 'TIME', 'SPC', 'PAGRID', 'NEST', 'I', 'J', 'K', 'INIT', 'CHEM', 'EMIS', 'PTEMIS', 'PIG', 'WADV', 'EADV', 'SADV', 'NADV', 'BADV', 'TADV', 'DIL', 'WDIF', 'EDIF', 'SDIF', 'NDIF', 'BDIF', 'TDIF', 'DDEP', 'WDEP', 'INORGACHEM', 'ORGACHEM', 'AQACHEM', 'FCONC', 'UCNV', 'AVOL', 'EPAD'],
formats=['>i', '>i', '>f', '>S10', '>i', '>i', '>i', '>i', '>i', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>f', '>i']
)
)
}
def __init__(self,rf,proc_dict=None,units='umol/m**3', oldnames = False, **props):
"""
Initialization included reading the header and learning
about the format.
see __readheader and __gettimestep() for more info
Keywords (i.e., props) for projection: P_ALP, P_BET, P_GAM, XCENT, YCENT, XORIG, YORIG, XCELL, YCELL
"""
if proc_dict!=None:
self.proc_dict=proc_dict
else:
self.proc_dict=None
self.__rffile=open(rf, 'rb')
self.__rffile.seek(0,2)
if self.__rffile.tell()<2147483648:
warn("For greater speed on files <2GB use ipr_memmap")
self.__rffile.seek(0,0)
self.units=units
self.__readheader()
self.__setDomain__()
self.__gettimestep()
tdim = self.createDimension('TSTEP',self.NSTEPS)
tdim.setunlimited(True)
self.createDimension('DATE-TIME',2)
self.createDimension('VAR',self.NSPCS*self.NPROCESS)
varkeys=["_".join([j[1],j[0]]) for j in [i for i in cartesian([i.strip() for i in self.spcnames['SPECIES'].tolist()],self.proc_dict.keys())]]+['TFLAG']
self.variables=PseudoNetCDFVariables(self.__variables,varkeys)
for k, v in props.items():
setattr(self, k, v)
add_cf_from_ioapi(self)
def __variables(self,proc_spc):
if proc_spc=='TFLAG':
time=self.variables['TIME_%s' % self.spcnames[0][1].strip()]
date=self.variables['DATE_%s' % self.spcnames[0][1].strip()]
self.variables['TFLAG']=PseudoNetCDFVariable(self,'proc_spc','i',('TSTEP','VAR','DATE-TIME'),values=ConvertCAMxTime(date[:,0,0,0],time[:,0,0,0],len(self.dimensions['VAR'])))
return self.variables['TFLAG']
self.variables.clear()
for k in self.proc_dict:
proc=proc_spc[:len(k)]
spc=proc_spc[len(k)+1:]
if proc==k and spc.ljust(10) in self.spcnames['SPECIES'].tolist():
spcprocs=self.__readalltime(spc)
for p,plong in self.proc_dict.items():
var_name=p+'_'+spc
# IPR units are consistent with 'IPR'
if p == 'UCNV':
units = 'm**3/mol'
elif p == 'AVOL':
units = 'm**3'
else:
units = get_uamiv_units('IPR', spc)
self.variables[var_name] = PseudoNetCDFVariable(self,var_name,'f',('TSTEP','LAY','ROW','COL'),
values=spcprocs[p],
units=units,
var_desc=(var_name).ljust(16),
long_name=(var_name).ljust(16))
del spcprocs
return self.variables[proc_spc]
raise KeyError("Bad!")
def __readonetime(self,ntime,spc):
newpos = self.__start(ntime,spc)
oldpos = self.__rffile.tell()
relmove = newpos - oldpos
self.__rffile.seek(relmove, 1)
return fromfile(self.__rffile,dtype=self.__ipr_record_type,count=self.__block3d).reshape(self.NROWS,self.NCOLS,self.NLAYS).swapaxes(0,2).swapaxes(1,2)
def __readalltime(self,spc):
out=zeros((self.NSTEPS, self.NLAYS, self.NROWS, self.NCOLS), dtype = self.__ipr_record_type)
for it in range(self.NSTEPS):
out[it] = self.__readonetime(it,spc)
return out
def __start(self,ntime,spc):
nspec=self.spcnames['SPECIES'].tolist().index(spc.ljust(10))
return self.__data_start_byte+(long(ntime)*self.__block4d+self.__block3d*nspec)*self.__ipr_record_type.itemsize
def __readheader(self):
"""
__readheader reads the header section of the ipr file
it initializes each header field (see CAMx Users Manual for a list)
as properties of the ipr class
"""
self.runmessage=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','RUNMESSAGE','EPAD'],formats=['>i','>80S','>i'])),count=1)['RUNMESSAGE']
dates=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','SDATE','STIME','EDATE','ETIME','EPAD'],formats=['>i','>i','>f','>i','>f','>i'])),count=1)
self.SDATE=dates['SDATE']+2000000
self.STIME=dates['STIME']
self.EDATE=dates['EDATE']+2000000
self.ETIME=dates['ETIME']
self.__grids=[]
self.NGRIDS=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','NGRIDS','EPAD'],formats=['>i']*3)),count=1)['NGRIDS']
for grid in range(self.NGRIDS):
self.__grids.append(
fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','orgx','orgy','ncol','nrow','xsize','ysize','EPAD'],formats=['>i','>i','>i','>i','>i','>i','>i','>i'])),count=1)
)
self.spcnames = []
self.NSPCS=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','NSPCS','EPAD'],formats=['>i','>i','>i'])),count=1)['NSPCS'][0]
self.spcnames=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','SPECIES','EPAD'],formats=['>i','>10S','>i'])),count=self.NSPCS)
self.padomains=[]
self.NPADOMAINS=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','NPADOMAINS','EPAD'],formats=['>i','>i','>i'])),count=1)['NPADOMAINS'][0]
self.__padomains=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','grid','istart','iend','jstart','jend','blay','tlay','EPAD'],formats=['>i','>i','>i','>i','>i','>i','>i','>i','>i'])),count=self.NPADOMAINS)
self.__activedomain=self.__padomains[0]
self.prcnames=[]
self.NPROCESS=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','NPRCS','EPAD'],formats=['>i','>i','>i'])),count=1)['NPRCS']
self.__ipr_record_type=self.__ipr_record_type[self.NPROCESS[0]]
if self.proc_dict is None:
self.proc_dict={
'INIT': 'Initial concentration',
'CHEM': 'Chemistry',
'EMIS': 'Area emissions',
'PTEMIS': 'Point source emissions',
'PIG': 'Plume-in-Grid change',
'WADV': 'West boundary advection',
'EADV': 'East boundary advection',
'SADV': 'South boundary advection',
'NADV': 'North boundary advection',
'BADV': 'Bottom boundary advection',
'TADV': 'Top boundary advection',
'DIL': 'Dilution in the vertical',
'WDIF': 'West boundary diffusion',
'EDIF': 'East boundary diffusion',
'SDIF': 'South boundary diffusion',
'NDIF': 'North boundary diffusion',
'BDIF': 'Bottom boundary diffusion',
'TDIF': 'Top boundary diffusion',
'DDEP': 'Dry deposition',
'WDEP': 'Wet deposition',
'FCONC': 'Final concentration',
'UCNV': 'Units conversion',
'AVOL': 'Average cell volume',
'DATE': 'DATE',
'TIME': 'TIME',
'K': 'K',
'J': 'J',
'I': 'I'
}
if self.NPROCESS[0] == 24:
self.proc_dict['AERCHEM'] = 'Aerosol chemistry'
elif self.NPROCESS[0] == 26:
self.proc_dict['INORGACHEM'] = 'Inorganic Aerosol chemistry'
self.proc_dict['ORGACHEM'] = 'Organic Aerosol chemistry'
self.proc_dict['AQACHEM'] = 'Aqueous Aerosol chemistry'
else:
warn('Unknown version; cannot add aerosol chemistry')
self.prcnames=fromfile(self.__rffile,dtype=dtype(dict(names=['SPAD','PROCESS','EPAD'],formats=['>i','>25S','>i'])),count=self.NPROCESS)
self.__data_start_byte=self.__rffile.tell()
def __setDomain__(self,id=0):
self.__activedomain=self.__padomains[id]
self.createDimension('COL',self.__activedomain['iend']-self.__activedomain['istart']+1)
self.createDimension('ROW',self.__activedomain['jend']-self.__activedomain['jstart']+1)
self.createDimension('LAY',self.__activedomain['tlay']-self.__activedomain['blay']+1)
self.NCOLS=len(self.dimensions['COL'])
self.NROWS=len(self.dimensions['ROW'])
self.NLAYS=len(self.dimensions['LAY'])
self.__block3d=self.NLAYS*self.NROWS*self.NCOLS
self.__block4d=self.__block3d*self.NSPCS
def __gettimestep(self):
"""
Header information provides start and end date, but does not
indicate the increment between. This routine reads the first
and second date/time and initializes variables indicating the
timestep length and the anticipated number.
"""
self.__rffile.seek(self.__data_start_byte,0)
temp=fromfile(self.__rffile,dtype=self.__ipr_record_type,count=len(self.dimensions['LAY'])*len(self.dimensions['ROW'])*len(self.dimensions['COL'])+1)
self.TSTEP=timediff((self.SDATE,self.STIME),(temp[-1]['DATE']+2000000,temp[-1]['TIME']))
self.NSTEPS=int(timediff((self.SDATE,self.STIME),(self.EDATE,self.ETIME))/self.TSTEP)
class cloud_rain(PseudoNetCDFFile):
"""
cloud_rain provides a PseudoNetCDF interface for CAMx
cloud_rain files. Where possible, the inteface follows
IOAPI conventions (see www.baronams.com).
ex:
>>> cloud_rain_path = 'cloud_rain.bin'
>>> rows, cols = 65, 83
>>> cloud_rainfile = cloud_rain(cloud_rain_path, rows, cols)
>>> cloud_rainfile.variables.keys()
['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG']
>>> v = cloud_rainfile.variables['CLOUD']
>>> tflag = cloud_rainfile.variables['TFLAG']
>>> tflag.dimensions
('TSTEP', 'VAR', 'DATE-TIME')
>>> tflag[0, 0, :]
array([2005185, 0])
>>> tflag[-1, 0, :]
array([2005185, 240000])
>>> v.dimensions
('TSTEP', 'LAY', 'ROW', 'COL')
>>> v.shape
(25, 28, 65, 83)
>>> cloud_rainfile.dimensions
{'TSTEP': 25, 'LAY': 28, 'ROW': 65, 'COL': 83}
"""
def __init__(self, rf, rows = None,cols = None):
f = open(rf, 'rb')
f.seek(0, 2)
flen = f.tell()
offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8
self.__memmap = memmap(rf, dtype = '>f', mode = 'r', offset = offset)
ncols, nrows, nlays = struct.unpack({35:'>i15ciiii', 40:'>i20ciiii'}[offset], open(rf, 'rb').read(offset))[-4:-1]
self.createDimension('COL', ncols)
self.createDimension('ROW', nrows)
self.createDimension('LAY', nlays)
header = np.fromfile(rf, dtype = {35:'>i4,S15,>i4,>i4,>i4,>i4,>i4,>f4,>i4', 40:'>i4,S20,>i4,>i4,>i4,>i4,>i4,>f4,>i4'}[offset], count = 1)[0]
self.FILEDESC = ''.join(header[1].decode())
self.STIME, self.SDATE = header.tolist()[-2:]
if self.SDATE < 10000:
self.SDATE+=2000000
if (ncols!=cols and cols!=None) or (rows!=rows and rows!=None):
warn('Files says cols = %d, rows = %d, and lays = %d; you said cols = %d and rows = %d' % (ncols, nrows, nlays, cols, rows))
self.createDimension('DATE-TIME', 2)
self.VERSION, varkeys = {35:('<4.3', ['CLOUD', 'PRECIP', 'COD', 'TFLAG']), 40:('4.3', ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'])}[offset]
self.createDimension('TSTEP', (flen - offset) // ((len(varkeys) - 1) * nlays * (nrows * ncols + 2) * 4 + 16))
self.createDimension('VAR', len(varkeys) - 1)
self.NVARS = len(self.dimensions['VAR'])
self.NLAYS = len(self.dimensions['LAY'])
self.NROWS = len(self.dimensions['ROW'])
self.NCOLS = len(self.dimensions['COL'])
self.FTYPE = 1
self.variables = PseudoNetCDFVariables(self.__var_get, varkeys)
self.SDATE,self.STIME = self.variables['TFLAG'][0, 0, :]
def __set_var(self, key, vals_idx):
times = len(self.dimensions['TSTEP'])
lays = len(self.dimensions['LAY'])
rows = len(self.dimensions['ROW'])
cols = len(self.dimensions['COL'])
v = PseudoNetCDFVariable(self, key, 'f', ('TSTEP', 'LAY', 'ROW', 'COL'), values = self.__memmap[vals_idx].reshape(times, lays, rows, cols))
v.units = {'COD':'None'}.get(key, 'g/m**3')
v.long_name = key
v.var_desc = key
self.variables[key] = v
def __var_get(self, key):
times = len(self.dimensions['TSTEP'])
rows = len(self.dimensions['ROW'])
cols = len(self.dimensions['COL'])
lays = len(self.dimensions['LAY'])
vars = len(list(self.variables.keys())) - 1
hour = 1
date = 2
cloud = 3
rain = 4
snow = 5
graupel = 6
cod = 7
stagger = 8
out_idx = zeros(self.__memmap.shape,dtype = 'b')
out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,1] = hour
out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,2] = date
self.variables['TFLAG'] = ConvertCAMxTime(self.__memmap[out_idx==date].view('>i'), self.__memmap[out_idx==hour], len(self.dimensions['VAR']))
val_shape = out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,4:].reshape(times, lays, vars, rows * cols + 2)[:, :,:,1:-1].reshape(times, lays, vars, rows, cols)
if self.VERSION=='<4.3':
val_shape[:, :,0, :, :] = cloud
val_shape[:, :,1, :, :] = rain
val_shape[:, :,2, :, :] = cod
self.__set_var('CLOUD', out_idx==cloud)
self.__set_var('PRECIP', out_idx==rain)
self.__set_var('COD', out_idx==cod)
else:
val_shape[:, :,0, :, :] = cloud
val_shape[:, :,1, :, :] = rain
val_shape[:, :,2, :, :] = snow
val_shape[:, :,3, :, :] = graupel
val_shape[:, :,4, :, :] = cod
self.__set_var('CLOUD', out_idx==cloud)
self.__set_var('RAIN', out_idx==rain)
self.__set_var('SNOW', out_idx==snow)
self.__set_var('GRAUPEL', out_idx==graupel)
self.__set_var('COD', out_idx==cod)
buf = self.__memmap[out_idx==0].reshape(vars * times * lays + times, 2)
if not (buf[:,0]==buf[:,1]).all():
raise ValueError("Buffer")
return self.variables[key]