class cloud_rain(PseudoNetCDFFile): """ cloud_rain provides a PseudoNetCDF interface for CAMx cloud_rain files. Where possible, the inteface follows IOAPI conventions (see www.baronams.com). ex: >>> cloud_rain_path = 'cloud_rain.bin' >>> rows, cols = 65, 83 >>> cloud_rainfile = cloud_rain(cloud_rain_path, rows, cols) >>> cloud_rainfile.variables.keys() ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'] >>> v = cloud_rainfile.variables['CLOUD'] >>> tflag = cloud_rainfile.variables['TFLAG'] >>> tflag.dimensions ('TSTEP', 'VAR', 'DATE-TIME') >>> tflag[0, 0, :] array([2005185, 0]) >>> tflag[-1, 0, :] array([2005185, 240000]) >>> v.dimensions ('TSTEP', 'LAY', 'ROW', 'COL') >>> v.shape (25, 28, 65, 83) >>> cloud_rainfile.dimensions {'TSTEP': 25, 'LAY': 28, 'ROW': 65, 'COL': 83} """ def __init__(self, rf, rows=None, cols=None): f = open(rf, 'rb') f.seek(0, 2) flen = f.tell() offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8 self.__memmap = memmap(rf, dtype='>f', mode='r', offset=offset) ncols, nrows, nlays = struct.unpack({ 35: '>i15ciiii', 40: '>i20ciiii' }[offset], open(rf, 'rb').read(offset))[-4:-1] self.createDimension('COL', ncols) self.createDimension('ROW', nrows) self.createDimension('LAY', nlays) header = np.fromfile(rf, dtype={ 35: '>i4,S15,>i4,>i4,>i4,>i4,>i4,>f4,>i4', 40: '>i4,S20,>i4,>i4,>i4,>i4,>i4,>f4,>i4' }[offset], count=1)[0] self.FILEDESC = ''.join(header[1].decode()) self.STIME, self.SDATE = header.tolist()[-2:] if self.SDATE < 10000: self.SDATE += 2000000 if (ncols != cols and cols != None) or (rows != rows and rows != None): warn( 'Files says cols = %d, rows = %d, and lays = %d; you said cols = %d and rows = %d' % (ncols, nrows, nlays, cols, rows)) self.createDimension('DATE-TIME', 2) self.VERSION, varkeys = { 35: ('<4.3', ['CLOUD', 'PRECIP', 'COD', 'TFLAG']), 40: ('4.3', ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG']) }[offset] self.createDimension('TSTEP', (flen - offset) // ((len(varkeys) - 1) * nlays * (nrows * ncols + 2) * 4 + 16)) self.createDimension('VAR', len(varkeys) - 1) self.NVARS = len(self.dimensions['VAR']) self.NLAYS = len(self.dimensions['LAY']) self.NROWS = len(self.dimensions['ROW']) self.NCOLS = len(self.dimensions['COL']) self.FTYPE = 1 self.variables = PseudoNetCDFVariables(self.__var_get, varkeys) self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :] def __set_var(self, key, vals_idx): times = len(self.dimensions['TSTEP']) lays = len(self.dimensions['LAY']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) v = PseudoNetCDFVariable(self, key, 'f', ('TSTEP', 'LAY', 'ROW', 'COL'), values=self.__memmap[vals_idx].reshape( times, lays, rows, cols)) v.units = {'COD': 'None'}.get(key, 'g/m**3') v.long_name = key v.var_desc = key self.variables[key] = v def __var_get(self, key): times = len(self.dimensions['TSTEP']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) lays = len(self.dimensions['LAY']) vars = len(list(self.variables.keys())) - 1 hour = 1 date = 2 cloud = 3 rain = 4 snow = 5 graupel = 6 cod = 7 stagger = 8 out_idx = zeros(self.__memmap.shape, dtype='b') out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:, 1] = hour out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:, 2] = date self.variables['TFLAG'] = ConvertCAMxTime( self.__memmap[out_idx == date].view('>i'), self.__memmap[out_idx == hour], len(self.dimensions['VAR'])) val_shape = out_idx.reshape( times, lays * vars * (rows * cols + 2) + 4)[:, 4:].reshape( times, lays, vars, rows * cols + 2)[:, :, :, 1:-1].reshape(times, lays, vars, rows, cols) if self.VERSION == '<4.3': val_shape[:, :, 0, :, :] = cloud val_shape[:, :, 1, :, :] = rain val_shape[:, :, 2, :, :] = cod self.__set_var('CLOUD', out_idx == cloud) self.__set_var('PRECIP', out_idx == rain) self.__set_var('COD', out_idx == cod) else: val_shape[:, :, 0, :, :] = cloud val_shape[:, :, 1, :, :] = rain val_shape[:, :, 2, :, :] = snow val_shape[:, :, 3, :, :] = graupel val_shape[:, :, 4, :, :] = cod self.__set_var('CLOUD', out_idx == cloud) self.__set_var('RAIN', out_idx == rain) self.__set_var('SNOW', out_idx == snow) self.__set_var('GRAUPEL', out_idx == graupel) self.__set_var('COD', out_idx == cod) buf = self.__memmap[out_idx == 0].reshape(vars * times * lays + times, 2) if not (buf[:, 0] == buf[:, 1]).all(): raise ValueError("Buffer") return self.variables[key]
class cloud_rain(PseudoNetCDFFile): """ cloud_rain provides a PseudoNetCDF interface for CAMx cloud_rain files. Where possible, the inteface follows IOAPI conventions (see www.baronams.com). ex: >>> cloud_rain_path = 'cloud_rain.bin' >>> rows, cols = 65, 83 >>> cloud_rainfile = cloud_rain(cloud_rain_path, rows, cols) >>> cloud_rainfile.variables.keys() ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'] >>> v = cloud_rainfile.variables['CLOUD'] >>> tflag = cloud_rainfile.variables['TFLAG'] >>> tflag.dimensions ('TSTEP', 'VAR', 'DATE-TIME') >>> tflag[0, 0, :] array([2005185, 0]) >>> tflag[-1, 0, :] array([2005185, 240000]) >>> v.dimensions ('TSTEP', 'LAY', 'ROW', 'COL') >>> v.shape (25, 28, 65, 83) >>> cloud_rainfile.dimensions {'TSTEP': 25, 'LAY': 28, 'ROW': 65, 'COL': 83} """ def __init__(self, rf, rows = None,cols = None): f = open(rf, 'rb') f.seek(0, 2) flen = f.tell() offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8 self.__memmap = memmap(rf, dtype = '>f', mode = 'r', offset = offset) ncols, nrows, nlays = struct.unpack({35:'>i15ciiii', 40:'>i20ciiii'}[offset], open(rf, 'rb').read(offset))[-4:-1] self.createDimension('COL', ncols) self.createDimension('ROW', nrows) self.createDimension('LAY', nlays) header = np.fromfile(rf, dtype = {35:'>i4,S15,>i4,>i4,>i4,>i4,>i4,>f4,>i4', 40:'>i4,S20,>i4,>i4,>i4,>i4,>i4,>f4,>i4'}[offset], count = 1)[0] self.FILEDESC = ''.join(header[1].decode()) self.STIME, self.SDATE = header.tolist()[-2:] if self.SDATE < 10000: self.SDATE+=2000000 if (ncols!=cols and cols!=None) or (rows!=rows and rows!=None): warn('Files says cols = %d, rows = %d, and lays = %d; you said cols = %d and rows = %d' % (ncols, nrows, nlays, cols, rows)) self.createDimension('DATE-TIME', 2) self.VERSION, varkeys = {35:('<4.3', ['CLOUD', 'PRECIP', 'COD', 'TFLAG']), 40:('4.3', ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'])}[offset] self.createDimension('TSTEP', (flen - offset) // ((len(varkeys) - 1) * nlays * (nrows * ncols + 2) * 4 + 16)) self.createDimension('VAR', len(varkeys) - 1) self.NVARS = len(self.dimensions['VAR']) self.NLAYS = len(self.dimensions['LAY']) self.NROWS = len(self.dimensions['ROW']) self.NCOLS = len(self.dimensions['COL']) self.FTYPE = 1 self.variables = PseudoNetCDFVariables(self.__var_get, varkeys) self.SDATE,self.STIME = self.variables['TFLAG'][0, 0, :] def __set_var(self, key, vals_idx): times = len(self.dimensions['TSTEP']) lays = len(self.dimensions['LAY']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) v = PseudoNetCDFVariable(self, key, 'f', ('TSTEP', 'LAY', 'ROW', 'COL'), values = self.__memmap[vals_idx].reshape(times, lays, rows, cols)) v.units = {'COD':'None'}.get(key, 'g/m**3') v.long_name = key v.var_desc = key self.variables[key] = v def __var_get(self, key): times = len(self.dimensions['TSTEP']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) lays = len(self.dimensions['LAY']) vars = len(list(self.variables.keys())) - 1 hour = 1 date = 2 cloud = 3 rain = 4 snow = 5 graupel = 6 cod = 7 stagger = 8 out_idx = zeros(self.__memmap.shape,dtype = 'b') out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,1] = hour out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,2] = date self.variables['TFLAG'] = ConvertCAMxTime(self.__memmap[out_idx==date].view('>i'), self.__memmap[out_idx==hour], len(self.dimensions['VAR'])) val_shape = out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:,4:].reshape(times, lays, vars, rows * cols + 2)[:, :,:,1:-1].reshape(times, lays, vars, rows, cols) if self.VERSION=='<4.3': val_shape[:, :,0, :, :] = cloud val_shape[:, :,1, :, :] = rain val_shape[:, :,2, :, :] = cod self.__set_var('CLOUD', out_idx==cloud) self.__set_var('PRECIP', out_idx==rain) self.__set_var('COD', out_idx==cod) else: val_shape[:, :,0, :, :] = cloud val_shape[:, :,1, :, :] = rain val_shape[:, :,2, :, :] = snow val_shape[:, :,3, :, :] = graupel val_shape[:, :,4, :, :] = cod self.__set_var('CLOUD', out_idx==cloud) self.__set_var('RAIN', out_idx==rain) self.__set_var('SNOW', out_idx==snow) self.__set_var('GRAUPEL', out_idx==graupel) self.__set_var('COD', out_idx==cod) buf = self.__memmap[out_idx==0].reshape(vars * times * lays + times, 2) if not (buf[:,0]==buf[:,1]).all(): raise ValueError("Buffer") return self.variables[key]
class cloud_rain(PseudoNetCDFFile): """ cloud_rain provides a PseudoNetCDF interface for CAMx cloud_rain files. Where possible, the inteface follows IOAPI conventions (see www.baronams.com). ex: >>> cloud_rain_path = 'cloud_rain.bin' >>> rows, cols = 65, 83 >>> cloud_rainfile = cloud_rain(cloud_rain_path, rows, cols) >>> cloud_rainfile.variables.keys() ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'] >>> v = cloud_rainfile.variables['CLOUD'] >>> tflag = cloud_rainfile.variables['TFLAG'] >>> tflag.dimensions ('TSTEP', 'VAR', 'DATE-TIME') >>> tflag[0, 0, :] array([2005185, 0]) >>> tflag[-1, 0, :] array([2005185, 240000]) >>> v.dimensions ('TSTEP', 'LAY', 'ROW', 'COL') >>> v.shape (25, 28, 65, 83) >>> cloud_rainfile.dimensions {'TSTEP': 25, 'LAY': 28, 'ROW': 65, 'COL': 83} """ def __init__(self, rf, rows=None, cols=None): f = open(rf, 'rb') f.seek(0, 2) flen = f.tell() offset = struct.unpack('>i', open(rf, 'rb').read(4))[0] + 8 self.__memmap = memmap(rf, dtype='>f', mode='r', offset=offset) cldhdrlen = offset - 20 line1fmt = '>i%dciiii' % cldhdrlen ncols, nrows, nlays = struct.unpack(line1fmt, open(rf, 'rb').read(offset))[-4:-1] self.createDimension('COL', ncols) self.createDimension('ROW', nrows) self.createDimension('LAY', nlays) mydt = '>i4,S%d,>i4,>i4,>i4,>i4,>i4,>f4,>i4' % cldhdrlen header = np.fromfile(rf, dtype=mydt, count=1)[0] self.FILEDESC = ''.join(header[1].decode()) self.STIME, self.SDATE = header.tolist()[-2:] if self.SDATE < 10000: self.SDATE += 2000000 if (((ncols != cols and cols is not None) or (rows != rows and rows is not None))): warn(('Files says cols = %d, rows = %d, and lays = %d; ' + 'you said cols = %d and rows = %d') % (ncols, nrows, nlays, cols, rows)) self.createDimension('DATE-TIME', 2) datasize = (flen - offset) # Try 5 first (contemporary) # Try 3 second (old) # end on 5 as a failsafe for nvars in [5, 3, 5]: timesize = (nvars * nlays * (nrows * ncols + 2) * 4 + 16) if (datasize % timesize) == 0: break else: warn('File appears incomplete using 3 (v4.2) or 5 ' + 'variables (>=v4.3); expected to fail') if nvars < 5: self.VERSION = '<4.3' varkeys = ['CLOUD', 'PRECIP', 'COD', 'TFLAG'] else: self.VERSION = '>=4.3' varkeys = ['CLOUD', 'RAIN', 'SNOW', 'GRAUPEL', 'COD', 'TFLAG'] ntimes = datasize // timesize self.createDimension('TSTEP', ntimes) self.createDimension('VAR', len(varkeys) - 1) self.NVARS = len(self.dimensions['VAR']) self.NLAYS = len(self.dimensions['LAY']) self.NROWS = len(self.dimensions['ROW']) self.NCOLS = len(self.dimensions['COL']) self.FTYPE = 1 self.variables = PseudoNetCDFVariables(self.__var_get, varkeys) self.SDATE, self.STIME = self.variables['TFLAG'][0, 0, :] def __set_var(self, key, vals_idx): times = len(self.dimensions['TSTEP']) lays = len(self.dimensions['LAY']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) vals = self.__memmap[vals_idx].reshape(times, lays, rows, cols) v = PseudoNetCDFVariable(self, key, 'f', ('TSTEP', 'LAY', 'ROW', 'COL'), values=vals) v.units = {'COD': 'None'}.get(key, 'g/m**3') v.long_name = key v.var_desc = key self.variables[key] = v def __var_get(self, key): times = len(self.dimensions['TSTEP']) rows = len(self.dimensions['ROW']) cols = len(self.dimensions['COL']) lays = len(self.dimensions['LAY']) vars = len(list(self.variables.keys())) - 1 hour = 1 date = 2 cloud = 3 rain = 4 snow = 5 graupel = 6 cod = 7 # stagger = 8 out_idx = zeros(self.__memmap.shape, dtype='b') out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:, 1] = hour out_idx.reshape(times, lays * vars * (rows * cols + 2) + 4)[:, 2] = date dateblock = self.__memmap[out_idx == date].view('>i') hourblock = self.__memmap[out_idx == hour] nvars = len(self.dimensions['VAR']) self.variables['TFLAG'] = ConvertCAMxTime(dateblock, hourblock, nvars) newshape1 = (times, lays * vars * (rows * cols + 2) + 4) newshape2 = (times, lays, vars, rows * cols + 2) newshape3 = (times, lays, vars, rows, cols) val_shape = out_idx.reshape(*newshape1)[:, 4:] val_shape = val_shape.reshape(*newshape2)[:, :, :, 1:-1] val_shape = val_shape.reshape(*newshape3) if self.VERSION == '<4.3': val_shape[:, :, 0, :, :] = cloud val_shape[:, :, 1, :, :] = rain val_shape[:, :, 2, :, :] = cod self.__set_var('CLOUD', out_idx == cloud) self.__set_var('PRECIP', out_idx == rain) self.__set_var('COD', out_idx == cod) else: val_shape[:, :, 0, :, :] = cloud val_shape[:, :, 1, :, :] = rain val_shape[:, :, 2, :, :] = snow val_shape[:, :, 3, :, :] = graupel val_shape[:, :, 4, :, :] = cod self.__set_var('CLOUD', out_idx == cloud) self.__set_var('RAIN', out_idx == rain) self.__set_var('SNOW', out_idx == snow) self.__set_var('GRAUPEL', out_idx == graupel) self.__set_var('COD', out_idx == cod) buf = self.__memmap[out_idx == 0].reshape(vars * times * lays + times, 2) if not (buf[:, 0] == buf[:, 1]).all(): raise ValueError("Buffer") return self.variables[key]