def test_24qd(self): import os, sys import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd g = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.) lvls = (500.,850.,1000.) v = vgd.vgd_new_pres(lvls) ip1list = vgd.vgd_get(v, 'VIPT') datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed'] npdtype = rmn.dtype_fst2numpy(datyp) rshape = (g['ni'], g['nj'], len(ip1list)) r = rmn.FST_RDE_META_DEFAULT.copy() r.update(g) r.update({ 'nomvar': 'MASK', 'nk' : len(ip1list), 'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000), 'ip2' : 6, 'deet' : 3600, 'npas' : 6, 'etiket': 'my_etk', 'datyp' : datyp, 'd' : np.empty(rshape, dtype=npdtype, order='FORTRAN') }) r['d'][:,:,:] = 0. r['d'][10:-11,5:-6,:] = 1. fileNameOut = 'newfromscratch.fst' fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) r2d = r.copy() for k in range(len(ip1list)): r2d.update({'nk':1, 'ip1':ip1list[k], 'd':np.asfortranarray(r['d'][:,:,k])}) rmn.fstecr(fileIdOut, r2d['d'], r2d) rmn.writeGrid(fileIdOut, g) vgd.vgd_write(v, fileIdOut) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_23qd(self): import os, sys, datetime from scipy.constants import knot as KNOT2MS import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' fileNameOut = 'uvfstfile.fst' fileIdIn = rmn.fstopenall(os.getenv('CMCGRIDF')+'/prog/regeta/'+fdate) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) vgd.vgd_write(vgd.vgd_read(fileIdIn), fileIdOut) (uu, vv, uvarray, copyGrid) = ({'d': None}, {'d': None}, None, True) for k in rmn.fstinl(fileIdIn, nomvar='UU'): uu = rmn.fstluk(k, dataArray=uu['d']) vv = rmn.fstlir(fileIdIn, nomvar='VV', ip1=uu['ip1'], ip2=uu['ip2'], datev=uu['datev'],dataArray=vv['d']) if uvarray is None: uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN') uv = uu.copy() uv.update({'d':uvarray, 'nomvar': 'WSPD'}) uv['d'][:,:] = np.sqrt(uu['d']**2. + vv['d']**2.) * KNOT2MS rmn.fstecr(fileIdOut, uv) if copyGrid: copyGrid = False rmn.writeGrid(fileIdOut, rmn.readGrid(fileIdIn, uu)) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_23qd(self): import os, sys, datetime from scipy.constants import knot as KNOT2MS import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' fileNameOut = 'uvfstfile.fst' fileIdIn = rmn.fstopenall( os.getenv('CMCGRIDF') + '/prog/regeta/' + fdate) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) vgd.vgd_write(vgd.vgd_read(fileIdIn), fileIdOut) (uu, vv, uvarray, copyGrid) = ({'d': None}, {'d': None}, None, True) for k in rmn.fstinl(fileIdIn, nomvar='UU'): uu = rmn.fstluk(k, dataArray=uu['d']) vv = rmn.fstlir(fileIdIn, nomvar='VV', ip1=uu['ip1'], ip2=uu['ip2'], datev=uu['datev'], dataArray=vv['d']) if uvarray is None: uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN') uv = uu.copy() uv.update({'d': uvarray, 'nomvar': 'WSPD'}) uv['d'][:, :] = np.sqrt(uu['d']**2. + vv['d']**2.) * KNOT2MS rmn.fstecr(fileIdOut, uv) if copyGrid: copyGrid = False rmn.writeGrid(fileIdOut, rmn.readGrid(fileIdIn, uu)) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def writeRec(fname, rec): """ Write the record data along with grid info to file Args: fname (str): Filename to to write to rec (dict): Record meta + data + grid info to write Returns: None """ print("+ Write %s to: %s" % (rec['nomvar'], fname)) # Open File try: funit = rmn.fstopenall(fname, rmn.FST_RW) except: raise rmn.FSTDError("Problem Opening file: %s" % fname) # Write rec meta + data try: rmn.fstecr(funit,rec['d'],rec) except: raise rmn.FSTDError("Problem writing %s record" % rec['nomvar']) # Write grid (if need be) writeRecGrid(rec, funit, fname) # Close File rmn.fstcloseall(funit) return
def test_writeGrid(self): ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES') file0 = os.path.join(ATM_MODEL_DFILES.strip(),'bcmk/geophy.fst') funit = rmn.fstopenall(file0) rec = rmn.fstlir(funit, nomvar='ME') grid0 = rmn.readGrid(funit, rec) rmn.fstcloseall(funit) grid1 = rmn.defGrid_L(180,60,0.,180.,1.,0.5) grid2 = rmn.defGrid_ZE(90,45,10.,11.,1.,0.5,0.,180.,1.,270.) grid3 = rmn.defGrid_YY(31,5,0.,180.,1.,270.) self.erase_testfile() myfile = self.fname funit = rmn.fstopenall(myfile, rmn.FST_RW) rmn.fstecr(funit,rec['d'],rec) rmn.writeGrid(funit, grid0) rmn.writeGrid(funit, grid1) rmn.writeGrid(funit, grid2) rmn.writeGrid(funit, grid3) rmn.fstcloseall(funit) funit = rmn.fstopenall(myfile, rmn.FST_RO) rec = rmn.fstlir(funit, nomvar='ME') grid0b = rmn.readGrid(funit, rec) rmn.fstcloseall(funit) self.erase_testfile() for k in grid0.keys(): if isinstance(grid0[k],np.ndarray): ok = np.any(np.abs(grid0b[k]-grid0[k]) > self.epsilon) self.assertFalse(ok, 'For k=%s, grid0b - grid0 = %s' % (k,str(np.abs(grid0b[k]-grid0[k])))) else: self.assertEqual(grid0b[k],grid0[k], 'For k=%s, expected:%s, got:%s' % (k, str(grid0[k]), str(grid0b[k])))
def WriteFld(self, outfile, outrecord): filename = outfile if not os.path.isfile(filename): # Check that the output file exists print 'The output file was not found:', filename quit() # Open the file if not rmn.isFST( filename ): # Check that the output file is a standard format file raise rmn.FSTDError("Not an FSTD file: %s " % filename) try: # Open the standard format file fstID = rmn.fstopenall(filename, rmn.FST_RW_OLD) print(fstID) except: raise rmn.FSTDError("File not found/readable: %s" % filename) # Write out the standard format record varname = outrecord['nomvar'] try: rmn.fstecr(fstID, outrecord) except: raise rmn.FSTDError( "Problem writing the record " + varname + " to file: %s" % filename) # Issue a warning message rmn.fstcloseall(fstID) # Close the standard format file return
def writeRec(fname, rec): """ Write the record data along with grid info to file Args: fname (str): Filename to to write to rec (dict): Record meta + data + grid info to write Returns: None """ print("+ Write %s to: %s" % (rec['nomvar'], fname)) # Open File try: funit = rmn.fstopenall(fname, rmn.FST_RW) except: raise rmn.FSTDError("Problem Opening file: %s" % fname) # Write rec meta + data try: rmn.fstecr(funit, rec['d'], rec) except: raise rmn.FSTDError("Problem writing %s record" % rec['nomvar']) # Write grid (if need be) writeRecGrid(rec, funit, fname) # Close File rmn.fstcloseall(funit) return
def test_6(self): """ Write a record Starting from the read a record example above we can change the data and meta before writing it as another record in the same file or in another file. See also: rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstinf rpnpy.librmn.fstd98.fstinl rpnpy.librmn.fstd98.fstprm rpnpy.librmn.fstd98.fstluk rpnpy.librmn.fstd98.fstlir rpnpy.librmn.fstd98.fstecr rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.FSTDError rpnpy.librmn.RMNError rpnpy.librmn.const """ import os, os.path, sys import numpy as np import rpnpy.librmn.all as rmn # open input file and read PR record ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES') fileName = os.path.join(ATM_MODEL_DFILES.strip(), 'bcmk/2009042700_012') try: fileId = rmn.fstopenall(fileName, rmn.FST_RO) except: raise rmn.FSTDError("Problem opening File: %s" % fileName) try: pr_rec = rmn.fstlir( fileId, nomvar='PR') # Read 1st record matching nomvar=PR except: sys.stdout.write("Problem reading record in File: %s" % fileName) finally: rmn.fstcloseall(fileId) # open output file and write record fileName = 'some_rpnstd_file.fst' try: fileId = rmn.fstopenall(fileName, rmn.FST_RW) except: raise rmn.FSTDError("Problem opening File: %s" % fileName) try: pr_data = pr_rec['d'] pr_data /= max(1.e-5, np.amax(pr_data)) pr_rec['nomvar'] = 'PRN1' rmn.fstecr(fileId, pr_data, pr_rec) except: sys.stdout.write("Problem writing record in File: %s" % fileName) finally: rmn.fstcloseall(fileId) # Erase test file os.unlink(fileName)
def test_6(self): """ Write a record Starting from the read a record example above we can change the data and meta before writing it as another record in the same file or in another file. See also: rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstinf rpnpy.librmn.fstd98.fstinl rpnpy.librmn.fstd98.fstprm rpnpy.librmn.fstd98.fstluk rpnpy.librmn.fstd98.fstlir rpnpy.librmn.fstd98.fstecr rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.FSTDError rpnpy.librmn.RMNError rpnpy.librmn.const """ import os, os.path, sys import numpy as np import rpnpy.librmn.all as rmn # open input file and read PR record ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES') fileName = os.path.join(ATM_MODEL_DFILES.strip(), 'bcmk/2009042700_012') try: fileId = rmn.fstopenall(fileName, rmn.FST_RO) except: raise rmn.FSTDError("Problem opening File: %s" % fileName) try: pr_rec = rmn.fstlir(fileId, nomvar='PR') # Read 1st record matching nomvar=PR except: sys.stdout.write("Problem reading record in File: %s" % fileName) finally: rmn.fstcloseall(fileId) # open output file and write record fileName = 'some_rpnstd_file.fst' try: fileId = rmn.fstopenall(fileName, rmn.FST_RW) except: raise rmn.FSTDError("Problem opening File: %s" % fileName) try: pr_data = pr_rec['d'] pr_data /= max(1.e-5, np.amax(pr_data)) pr_rec['nomvar'] = 'PRN1' rmn.fstecr(fileId, pr_data, pr_rec) except: sys.stdout.write("Problem writing record in File: %s" % fileName) finally: rmn.fstcloseall(fileId) # Erase test file os.unlink(fileName)
def test_22(self): """ Edit: Copy records (a la editfst desire) This example shows how to * select records in a RPNStd file * read the record data + meta * write the record data + meta See also: rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrinl rpnpy.librmn.fstd98.fsrluk rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.const """ import os, os.path, sys import rpnpy.librmn.all as rmn ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES').strip() fileNameIn = os.path.join(ATM_MODEL_DFILES, 'bcmk') fileNameOut = 'myfstfile.fst' # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Get the list of records to copy keylist1 = rmn.fstinl(fileIdIn, nomvar='UU') keylist2 = rmn.fstinl(fileIdIn, nomvar='VV') for k in keylist1 + keylist2: # Read record data and meta from fileNameIn r = rmn.fstluk(k) # Write the record to fileNameOut rmn.fstecr(fileIdOut, r) print("CB22: Copied %s ip1=%d, ip2=%d, dateo=%s" % (r['nomvar'], r['ip1'], r['ip2'], r['dateo'])) except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_22(self): """ Edit: Copy records (a la editfst desire) This example shows how to * select records in a RPNStd file * read the record data + meta * write the record data + meta See also: rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrinl rpnpy.librmn.fstd98.fsrluk rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.const """ import os, os.path, sys import rpnpy.librmn.all as rmn ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES').strip() fileNameIn = os.path.join(ATM_MODEL_DFILES,'bcmk') fileNameOut = 'myfstfile.fst' # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Get the list of records to copy keylist1 = rmn.fstinl(fileIdIn, nomvar='UU') keylist2 = rmn.fstinl(fileIdIn, nomvar='VV') for k in keylist1+keylist2: # Read record data and meta from fileNameIn r = rmn.fstluk(k) # Write the record to fileNameOut rmn.fstecr(fileIdOut, r) print("CB22: Copied %s ip1=%d, ip2=%d, dateo=%s" % (r['nomvar'], r['ip1'], r['ip2'], r['dateo'])) except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_22qd(self): import os, os.path import rpnpy.librmn.all as rmn ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES').strip() fileNameOut = 'myfstfile.fst' fileIdIn = rmn.fstopenall(ATM_MODEL_DFILES+'/bcmk') fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) for k in rmn.fstinl(fileIdIn, nomvar='UU') + rmn.fstinl(fileIdIn, nomvar='VV'): rmn.fstecr(fileIdOut, rmn.fstluk(k)) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_22qd(self): import os, os.path import rpnpy.librmn.all as rmn ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES').strip() fileNameOut = 'myfstfile.fst' fileIdIn = rmn.fstopenall(ATM_MODEL_DFILES + '/bcmk') fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) for k in rmn.fstinl(fileIdIn, nomvar='UU') + rmn.fstinl(fileIdIn, nomvar='VV'): rmn.fstecr(fileIdOut, rmn.fstluk(k)) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_fstecr_fstluk_order(self): rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST) fname = '__rpnstd__testfile2__.fst' try: os.unlink(fname) except: pass funit = rmn.fstopenall(fname, rmn.FST_RW) (ig1, ig2, ig3, ig4) = rmn.cxgaig(self.grtyp, self.xg14[0], self.xg14[1], self.xg14[2], self.xg14[3]) (ni, nj) = (90, 45) la = rmn.FST_RDE_META_DEFAULT.copy() la.update({ 'nomvar': 'LA', 'typvar': 'C', 'ni': ni, 'nj': nj, 'nk': 1, 'grtyp': self.grtyp, 'ig1': ig1, 'ig2': ig2, 'ig3': ig3, 'ig4': ig4 }) lo = la.copy() lo['nomvar'] = 'LO' #Note: For the order to be ok in the FSTD file, order='FORTRAN' is mandatory la['d'] = np.empty((ni, nj), dtype=np.float32, order='FORTRAN') lo['d'] = np.empty((ni, nj), dtype=np.float32, order='FORTRAN') for j in range(nj): for i in range(ni): lo['d'][i, j] = 100. + float(i) la['d'][i, j] = float(j) rmn.fstecr(funit, la['d'], la) rmn.fstecr(funit, lo) rmn.fstcloseall(funit) funit = rmn.fstopenall(fname, rmn.FST_RW) kla = rmn.fstinf(funit, nomvar='LA')['key'] la2 = rmn.fstluk(kla) #,rank=2) klo = rmn.fstinf(funit, nomvar='LO')['key'] lo2 = rmn.fstluk(klo) #,rank=2) rmn.fstcloseall(funit) try: os.unlink(fname) except: pass self.assertTrue(np.isfortran(la2['d'])) self.assertTrue(np.isfortran(lo2['d'])) self.assertFalse(np.any(np.fabs(la2['d'] - la['d']) > self.epsilon)) self.assertFalse(np.any(np.fabs(lo2['d'] - lo['d']) > self.epsilon))
def test_24qd(self): import os, sys import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd g = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.) lvls = (500., 850., 1000.) v = vgd.vgd_new_pres(lvls) ip1list = vgd.vgd_get(v, 'VIPT') datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed'] npdtype = rmn.dtype_fst2numpy(datyp) rshape = (g['ni'], g['nj'], len(ip1list)) r = rmn.FST_RDE_META_DEFAULT.copy() r.update(g) r.update({ 'nomvar': 'MASK', 'nk': len(ip1list), 'dateo': rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000), 'ip2': 6, 'deet': 3600, 'npas': 6, 'etiket': 'my_etk', 'datyp': datyp, 'd': np.empty(rshape, dtype=npdtype, order='FORTRAN') }) r['d'][:, :, :] = 0. r['d'][10:-11, 5:-6, :] = 1. fileNameOut = 'newfromscratch.fst' fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) r2d = r.copy() for k in range(len(ip1list)): r2d.update({ 'nk': 1, 'ip1': ip1list[k], 'd': np.asfortranarray(r['d'][:, :, k]) }) rmn.fstecr(fileIdOut, r2d['d'], r2d) rmn.writeGrid(fileIdOut, g) vgd.vgd_write(v, fileIdOut) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_fstecr_fstluk_order(self): rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) fname = '__rpnstd__testfile2__.fst' try: os.unlink(fname) except: pass funit = rmn.fstopenall(fname,rmn.FST_RW) (ig1,ig2,ig3,ig4) = rmn.cxgaig(self.grtyp,self.xg14[0],self.xg14[1],self.xg14[2],self.xg14[3]) (ni,nj) = (90,45) la = rmn.FST_RDE_META_DEFAULT.copy() la.update( {'nomvar' : 'LA', 'typvar' : 'C', 'ni' : ni, 'nj' : nj, 'nk' : 1, 'grtyp' : self.grtyp, 'ig1' : ig1, 'ig2' : ig2, 'ig3' : ig3, 'ig4' : ig4 } ) lo = la.copy() lo['nomvar'] = 'LO' #Note: For the order to be ok in the FSTD file, order='FORTRAN' is mandatory la['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN') lo['d'] = np.empty((ni,nj),dtype=np.float32,order='FORTRAN') for j in xrange(nj): for i in xrange(ni): lo['d'][i,j] = 100.+float(i) la['d'][i,j] = float(j) rmn.fstecr(funit,la['d'],la) rmn.fstecr(funit,lo) rmn.fstcloseall(funit) funit = rmn.fstopenall(fname,rmn.FST_RW) kla = rmn.fstinf(funit,nomvar='LA')['key'] la2 = rmn.fstluk(kla)#,rank=2) klo = rmn.fstinf(funit,nomvar='LO')['key'] lo2 = rmn.fstluk(klo)#,rank=2) rmn.fstcloseall(funit) try: os.unlink(fname) except: pass self.assertTrue(np.isfortran(la2['d'])) self.assertTrue(np.isfortran(lo2['d'])) self.assertFalse(np.any(np.fabs(la2['d'] - la['d']) > self.epsilon)) self.assertFalse(np.any(np.fabs(lo2['d'] - lo['d']) > self.epsilon))
def writeRecGrid(rec, funit, fname=''): """ Write the record grid info to previously opened file Args: rec (dict): Record meta + grid info to write funit (int): Unit number the the opened file to write to fname (str): (optional) Filename to to write to Returns: None """ print("+ Write grid for %s to: %s" % (rec['nomvar'], fname)) rec2 = rmn.FST_RDE_META_DEFAULT.copy() rec2.update(rec) rec2['ip1'] = rec['grid']['tag1'] rec2['ip2'] = rec['grid']['tag2'] rec2['ip3'] = rec['grid']['tag3'] rec2['grtyp'] = rec['grid']['grref'] rec2['grref'] = rec['grid']['grref'] rec2['ig1'] = rec['grid']['ig1ref'] rec2['ig2'] = rec['grid']['ig2ref'] rec2['ig3'] = rec['grid']['ig3ref'] rec2['ig4'] = rec['grid']['ig4ref'] for k in ('ig1ref','ig2ref','ig3ref','ig4ref'): rec2[k] = rec['grid'][k] rec2['d'] = rec['grid']['ax'] rec2['ni'] = rec['grid']['ax'].shape[0] rec2['nj'] = rec['grid']['ax'].shape[1] rec2['shape'] = rec['grid']['ax'].shape rec2['nomvar'] = '>>' try: rmn.fstecr(funit,rec2['d'],rec2) except: raise rmn.FSTDError("Problem writing %s record" % rec2['nomvar']) rec2['d'] = rec['grid']['ay'] rec2['ni'] = rec['grid']['ay'].shape[0] rec2['nj'] = rec['grid']['ay'].shape[1] rec2['shape'] = rec['grid']['ay'].shape rec2['nomvar'] = '^^' try: rmn.fstecr(funit,rec2['d'],rec2) except: raise rmn.FSTDError("Problem writing %s record" % rec2['nomvar']) return
def writeRecGrid(rec, funit, fname=''): """ Write the record grid info to previously opened file Args: rec (dict): Record meta + grid info to write funit (int): Unit number the the opened file to write to fname (str): (optional) Filename to to write to Returns: None """ print("+ Write grid for %s to: %s" % (rec['nomvar'], fname)) rec2 = rmn.FST_RDE_META_DEFAULT.copy() rec2.update(rec) rec2['ip1'] = rec['grid']['tag1'] rec2['ip2'] = rec['grid']['tag2'] rec2['ip3'] = rec['grid']['tag3'] rec2['grtyp'] = rec['grid']['grref'] rec2['grref'] = rec['grid']['grref'] rec2['ig1'] = rec['grid']['ig1ref'] rec2['ig2'] = rec['grid']['ig2ref'] rec2['ig3'] = rec['grid']['ig3ref'] rec2['ig4'] = rec['grid']['ig4ref'] for k in ('ig1ref', 'ig2ref', 'ig3ref', 'ig4ref'): rec2[k] = rec['grid'][k] rec2['d'] = rec['grid']['ax'] rec2['ni'] = rec['grid']['ax'].shape[0] rec2['nj'] = rec['grid']['ax'].shape[1] rec2['shape'] = rec['grid']['ax'].shape rec2['nomvar'] = '>>' try: rmn.fstecr(funit, rec2['d'], rec2) except: raise rmn.FSTDError("Problem writing %s record" % rec2['nomvar']) rec2['d'] = rec['grid']['ay'] rec2['ni'] = rec['grid']['ay'].shape[0] rec2['nj'] = rec['grid']['ay'].shape[1] rec2['shape'] = rec['grid']['ay'].shape rec2['nomvar'] = '^^' try: rmn.fstecr(funit, rec2['d'], rec2) except: raise rmn.FSTDError("Problem writing %s record" % rec2['nomvar']) return
def createAFCDict(fileID, keyList1, keyList2, spc): ''' Creates a dictionary FCDict in the format of: FCDict{dateo:{'meta':metadata, 'vals':list([data],[data])}, dateo2:{..}...} ''' FCDict = {} for key1 in keyList1: meta, cData, dateo = getMeta(fileID, key1) # save this existing information meta['nomvar'] = '{}F'.format(spc) meta['dateo'] = dateo rmn.fstecr(fileID, cData, meta) # update the nomvar again for the combo meta['nomvar'] = '{}FC'.format(spc) if dateo not in FCDict.keys(): try: FCDict[dateo] = {'meta': meta, 'vals': [cData]} except: print('Did not add data for dateo {} in key1s'.format(dateo)) else: print('there are two keys for dateo {} in key1s'.format(dateo)) delRec(key1) for key2 in keyList2: meta2, cData2, dateo2 = getMeta(fileID, key2) # save the existing info meta2['nomvar'] = '{}C'.format(spc) meta2['dateo'] = dateo2 rmn.fstecr(fileID, cData2, meta2) meta2['nomvar'] = '{}FC'.format(spc) if dateo2 not in FCDict.keys(): try: FCDict[dateo2] = {'meta': meta2, 'vals': [cData2]} except: print('Did not add data for dateo {} in keys2'.format(dateo2)) elif len(FCDict[dateo2]['vals']) < 2: try: FCDict[dateo2]['vals'] += [cData2] except: print( 'Did not add arrays for dateo {} in keys2'.format(dateo2)) else: print('there are two keys for dateo {} in keys2'.format(dateo2)) delRec(key2) return FCDict
def smooth_my_field(fid_in, fid_out, varname, outname, etiket, grid_space, edge_value): """ Read, smooth and write field in file Args: fid_in (int) : Input file unit id fid_out (int) : Output file unit id varname (str) : input varname of field to interpolate outname (str) : varname of smoothed field in output file etiket (str) : etiket of smoothed field in output file grid_space (int) : Smoothing factor (nb of grid points) edge_value (float): Value to put a field edges Returns: None """ print("+ Read, smooth and write field: %s" % varname) # Get list of records try: klist = rmn.fstinl(fid_in, nomvar=varname) except: raise rmn.RMNError('Problem getting list of records for ' + varname) # read, smooth and write field for k in klist: try: myfield = rmn.fstluk(k) except: raise rmn.RMNError('Problem in reading var: ' + varname) try: myfield['d'] = smooth_my_data(myfield['d'], grid_space, edge_value) except: raise rmn.RMNError('Problem in smoothing var: ' + varname) try: myfield['nomvar'] = outname myfield['etiket'] = etiket rmn.fstecr(fid_out, myfield['d'], myfield) except: raise rmn.RMNError('Problem in writing var: %s at ip1=%d\n' % (outname, myfield['ip1'])) del myfield['d'], myfield return
def smooth_my_field(fid_in, fid_out, varname, outname, etiket, grid_space, edge_value): """ Read, smooth and write field in file Args: fid_in (int) : Input file unit id fid_out (int) : Output file unit id varname (str) : input varname of field to interpolate outname (str) : varname of smoothed field in output file etiket (str) : etiket of smoothed field in output file grid_space (int) : Smoothing factor (nb of grid points) edge_value (float): Value to put a field edges Returns: None """ print("+ Read, smooth and write field: %s" % varname) # Get list of records try: klist = rmn.fstinl(fid_in, nomvar=varname) except: raise rmn.RMNError('Problem getting list of records for '+varname) # read, smooth and write field for k in klist: try: myfield = rmn.fstluk(k) except: raise rmn.RMNError('Problem in reading var: '+varname) try: myfield['d'] = smooth_my_data(myfield['d'], grid_space, edge_value) except: raise rmn.RMNError('Problem in smoothing var: '+varname) try: myfield['nomvar'] = outname myfield['etiket'] = etiket rmn.fstecr(fid_out, myfield['d'], myfield) except: raise rmn.RMNError('Problem in writing var: %s at ip1=%d\n' % (outname,myfield['ip1'])) del myfield['d'], myfield return
def test_41qd(self): import os, sys, datetime import rpnpy.librmn.all as rmn fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameOut = 'p0fstfileqd.fst' fileIdIn = rmn.fstopenall(os.getenv('CMCGRIDF')+'/prog/regeta/'+fdate) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) gOut = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.) r = rmn.fstlir(fileIdIn, nomvar='P0') gIn = rmn.readGrid(fileIdIn, r) rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR) d = rmn.ezsint(gOut, gIn, r) r2 = r.copy() r2.update(gOut) r2.update({'etiket':'my_etk', 'd':d}) rmn.fstecr(fileIdOut, r2) rmn.writeGrid(fileIdOut, gOut) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def create_basefile(self): """create a basic test file for RPNFile tests""" self.erase_testfile() funit = rmn.fstopenall(self.fname, rmn.FST_RW) (ig1, ig2, ig3, ig4) = rmn.cxgaig(self.grtyp, self.xg14[0], self.xg14[1], self.xg14[2], self.xg14[3]) self.la = rmn.FST_RDE_META_DEFAULT.copy() self.la.update({ 'nomvar': 'LA', 'typvar': 'C', 'ni': self.lad.shape[0], 'nj': self.lad.shape[1], 'nk': 1, 'grtyp': self.grtyp, 'ig1': ig1, 'ig2': ig2, 'ig3': ig3, 'ig4': ig4, 'd': self.lad }) rmn.fstecr(funit, self.la['d'], self.la) self.lo = rmn.FST_RDE_META_DEFAULT.copy() self.lo.update({ 'nomvar': 'LO', 'typvar': 'C', 'ni': self.lod.shape[0], 'nj': self.lod.shape[1], 'nk': 1, 'grtyp': self.grtyp, 'ig1': ig1, 'ig2': ig2, 'ig3': ig3, 'ig4': ig4, 'd': self.lod }) rmn.fstecr(funit, self.lo['d'], self.lo) rmn.fstcloseall(funit) return (self.la, self.lo)
def test_41qd(self): import os, sys, datetime import rpnpy.librmn.all as rmn fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameOut = 'p0fstfileqd.fst' fileIdIn = rmn.fstopenall( os.getenv('CMCGRIDF') + '/prog/regeta/' + fdate) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) gOut = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.) r = rmn.fstlir(fileIdIn, nomvar='P0') gIn = rmn.readGrid(fileIdIn, r) rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR) d = rmn.ezsint(gOut, gIn, r) r2 = r.copy() r2.update(gOut) r2.update({'etiket': 'my_etk', 'd': d}) rmn.fstecr(fileIdOut, r2) rmn.writeGrid(fileIdOut, gOut) rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def create_basefile(self): """create a basic test file for RPNFile tests""" self.erase_testfile() funit = rmn.fstopenall(self.fname,rmn.FST_RW) (ig1,ig2,ig3,ig4) = rmn.cxgaig(self.grtyp,self.xg14[0],self.xg14[1],self.xg14[2],self.xg14[3]) self.la = rmn.FST_RDE_META_DEFAULT.copy() self.la.update( {'nomvar' : 'LA', 'typvar' : 'C', 'ni' : self.lad.shape[0], 'nj' : self.lad.shape[1], 'nk' : 1, 'grtyp' : self.grtyp, 'ig1' : ig1, 'ig2' : ig2, 'ig3' : ig3, 'ig4' : ig4, 'd' : self.lad} ) rmn.fstecr(funit,self.la['d'],self.la) self.lo = rmn.FST_RDE_META_DEFAULT.copy() self.lo.update( {'nomvar' : 'LO', 'typvar' : 'C', 'ni' : self.lod.shape[0], 'nj' : self.lod.shape[1], 'nk' : 1, 'grtyp' : self.grtyp, 'ig1' : ig1, 'ig2' : ig2, 'ig3' : ig3, 'ig4' : ig4, 'd' : self.lod} ) rmn.fstecr(funit,self.lo['d'],self.lo) rmn.fstcloseall(funit) return (self.la,self.lo)
def test_writeGrid(self): ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES') file0 = os.path.join(ATM_MODEL_DFILES.strip(), 'bcmk/geophy.fst') funit = rmn.fstopenall(file0) rec = rmn.fstlir(funit, nomvar='ME') grid0 = rmn.readGrid(funit, rec) rmn.fstcloseall(funit) grid1 = rmn.defGrid_L(180, 60, 0., 180., 1., 0.5) grid2 = rmn.defGrid_ZE(90, 45, 10., 11., 1., 0.5, 0., 180., 1., 270.) grid3 = rmn.defGrid_YY(31, 5, 0., 180., 1., 270.) self.erase_testfile() myfile = self.fname funit = rmn.fstopenall(myfile, rmn.FST_RW) rmn.fstecr(funit, rec['d'], rec) rmn.writeGrid(funit, grid0) rmn.writeGrid(funit, grid1) rmn.writeGrid(funit, grid2) rmn.writeGrid(funit, grid3) rmn.fstcloseall(funit) funit = rmn.fstopenall(myfile, rmn.FST_RO) rec = rmn.fstlir(funit, nomvar='ME') grid0b = rmn.readGrid(funit, rec) rmn.fstcloseall(funit) self.erase_testfile() for k in grid0.keys(): if isinstance(grid0[k], np.ndarray): ok = np.any(np.abs(grid0b[k] - grid0[k]) > self.epsilon) self.assertFalse( ok, 'For k=%s, grid0b - grid0 = %s' % (k, str(np.abs(grid0b[k] - grid0[k])))) else: self.assertEqual( grid0b[k], grid0[k], 'For k=%s, expected:%s, got:%s' % (k, str(grid0[k]), str(grid0b[k])))
def fstecr(data, iunit, nomvar, typvar, etiket, ip1, ip2, ip3, dateo, grtyp, ig1, ig2, ig3, ig4, deet, npas, nbits, datyp): """Write record data & meta(params) to file (Interface to fstecr), always append (no overwrite) Fstdc.fstecr(data, iunit, nomvar, typvar, etiket, ip1, ip2, ip3, dateo, grtyp, ig1, ig2, ig3, ig4, deet, npas, nbits, datyp) @param data array to be written to file (numpy.ndarray) @param iunit file unit number handle returned by Fstdc_fstouv (int) @param ... @exception TypeError @exception Fstdc.error """ try: ni = data.shape[0] (nj, nk) = (1, 1) if len(data.shape) > 1: nj = data.shape[1] if len(data.shape) > 2: nk = data.shape[2] except: raise error("fstecr: cannot get data shape") mymeta = { 'dateo' : dateo, 'deet' : deet, 'npas' : npas, 'ni' : ni, 'nj' : nj, 'nk' : nk, 'nbits' : nbits, 'datyp' : datyp, 'ip1' : ip1, 'ip2' : ip2, 'ip3' : ip3, 'typvar': typvar, 'nomvar': nomvar, 'etiket': etiket, 'grtyp' : grtyp, 'ig1' : ig1, 'ig2' : ig2, 'ig3' : ig3, 'ig4' : ig4 } try: return _rmn.fstecr(iunit, data, mymeta, rewrite=False) except _rmn.FSTDError: raise error('Problem writing rec param/meta')
def plotFSTs(season=season, spcs=spcs, spcsFiles=spcsFiles, outputName = outputName, saveDir=saveDir): # print minimum outputs rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) mInds = [] for m in season: mInds += [monthList.index(m)] if os.path.exists(saveDir) == False: nu.makeDir(saveDir) for spcInd, nomvar in enumerate(spcs): try: filename = os.path.join(saveDir, 'output_file_{0}_{1}.fst'.format(outputName, nomvar)) print('Creating and saving to {}'.format(filename)) tmp = open(filename, 'w+'); tmp.close() output_file = filename file_id = rmn.fnom(output_file) open_fst = rmn.fstouv(file_id, rmn.FST_RW) open_file = spcsFiles[spcInd] print "Parameter: " + nomvar seaSpcData = get_var(pyg.open(open_file), nomvar, mInds) nc_lnsp = pyg.open(lnsp_file) pressures = get_pressures(nc_lnsp, mInds) timelen, levlen, latlen, lonlen = seaSpcData.shape #NOTE: uncomment the following three lines to prep data for basemap use #lonShiftSSData = shift_lon(seaSpcData) #vertInterpSSData = vert_interp(pressures, lonShiftSSData) #meanSSData = np.mean(vertInterpSSData, axis=0) #NOTE: uncommment the following four liness to use for fst plotting vertInterpSSData = vert_interp(pressures, seaSpcData) meanSSData = np.mean(vertInterpSSData, axis=0) # temp for lvl, ray in enumerate(meanSSData): meanSSData[lvl] = np.flipud(ray) scaleFac = scaleSpcs[allSpcs.index(nomvar)] scaledSSData = meanSSData*scaleFac #define grid for this file - note that the MACC grid in the file is #defined for lons -180 to 180, but the python defGrid_L can't deal #with that and defines the grid from 0 to 360 so will have to reorder #the MACC fields a bit, or they end up 180 deg out of phase # Also necessary to add one more longitude to wrap around dlatlon = 360./lonlen # this is equal to the resolution of the grid params0 = { 'grtyp' : 'Z', 'grref' : 'L', 'nj' : latlen, 'ni' : lonlen, 'lat0' : -90., 'lon0' : 0., 'dlat' : dlatlon, 'dlon' : dlatlon } MACC_grid= rmn.encodeGrid(params0) print("Grids created.") print 'Grid Shape:' + str(MACC_grid['shape']) # copies the default record new_record = rmn.FST_RDE_META_DEFAULT.copy() tic_record = rmn.FST_RDE_META_DEFAULT.copy() tac_record = rmn.FST_RDE_META_DEFAULT.copy() try: rmn.writeGrid(file_id, MACC_grid) tac = rmn.fstinl(file_id, nomvar='>>')[0] tic = rmn.fstinl(file_id, nomvar='^^')[0] tic_record.update(rmn.fstprm(tic)) tac_record.update(rmn.fstprm(tac)) tic_record.update({'datyp' : rmn.FST_DATYP_LIST['float']}) tac_record.update({'datyp' : rmn.FST_DATYP_LIST['float']}) rmn.fsteff(tic) rmn.fsteff(tac) tic_record.update({'d': MACC_grid['ay']}) tac_record.update({'d': MACC_grid['ax']}) toc_record = vgd.vgd_new_pres(const_pressure, ip1=MACC_grid['ig1'], ip2=MACC_grid['ig2']) rmn.fstecr(file_id, tic_record) # write the dictionary record to the file as a new record rmn.fstecr(file_id, tac_record) # write the dictionary record to the file as a new record vgd.vgd_write(toc_record, file_id) except: raise for rp1 in xrange(len(const_pressure)): # writes a record for every level (as a different ip1) try: # converts rp1 into a ip1 with pressure kind ip1 = rmn.convertIp(rmn.CONVIP_ENCODE, const_pressure[rp1], rmn.KIND_PRESSURE) new_record.update(MACC_grid) new_record.update({ # Update with specific meta 'nomvar': nomvar, 'typvar': 'C', 'etiket': 'MACCRean', 'ni' : MACC_grid['ni'], 'nj' : MACC_grid['nj'], 'ig1' : tic_record['ip1'], 'ig2' : tic_record['ip2'], 'ig3' : tic_record['ip3'], 'ig4' : tic_record['ig4'], 'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20120101, 0000000), 'deet' : 0, # Timestep in sec 'ip1' : ip1 }) #tmp_nparray = np.asfortranarray(monthly_mean[rp1]) tmp = scaledSSData[rp1] tmp = np.transpose(tmp) # data array is structured as tmp = monthly_mean[level] where monthly_mean is [level, lat, lon] new_record.update({'d': tmp.astype(np.float32)}) # Updates with data array in the form (lon x lat) print "Defined a new record with dimensions ({0}, {1})".format(new_record['ni'], new_record['nj']) rmn.fstecr(file_id, new_record) # write the dictionary record to the file as a new record except: #rmn.closeall(file_id) rmn.fstfrm(file_id) rmn.fclos(file_id) raise rmn.fstfrm(file_id) rmn.fclos(file_id) print('{} complete~'.format(filename)) except: rmn.fstfrm(file_id) rmn.fclos(file_id) raise print('Finished plotting all FSTs. ')
def test_41(self): """ Horizontal Interpolation See also: """ import os, sys, datetime import rpnpy.librmn.all as rmn fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate) fileNameOut = 'p0fstfile.fst' # Restrict to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST) try: # Create Destination grid # Note: Destination grid can also be read from a file gp = { 'grtyp': 'Z', 'grref': 'E', 'ni': 90, 'nj': 45, 'lat0': 35., 'lon0': 250., 'dlat': 0.5, 'dlon': 0.5, 'xlat1': 0., 'xlon1': 180., 'xlat2': 1., 'xlon2': 270. } gOut = rmn.encodeGrid(gp) print("CB41: Defined a %s/%s grid of shape=%d, %d" % (gOut['grtyp'], gOut['grref'], gOut['ni'], gOut['nj'])) except: sys.stderr.write("Problem creating grid\n") sys.exit(1) # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Find and read record to interpolate with its grid r = rmn.fstlir(fileIdIn, nomvar='P0') gIn = rmn.readGrid(fileIdIn, r) print("CB41: Read P0") # Set interpolation options and interpolate rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR) d = rmn.ezsint(gOut, gIn, r) print("CB41: Interpolate P0") # Create new record to write with interpolated data and r2 = r.copy() # Preserve meta from original record r2.update(gOut) # update grid information r2.update({ # attach data and update specific meta 'etiket': 'my_etk', 'd': d }) # Write record data + meta + grid to file rmn.fstecr(fileIdOut, r2) rmn.writeGrid(fileIdOut, gOut) print("CB41: Wrote interpolated P0 and its grid") except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def updateFST(fileID, AFCDict): """ Updates the original FST file with the new PM data and corresponding metadata. """ for ip1 in AFCDict: rmn.fstecr(fileID, AFCDict[ip1]['vals'], AFCDict[ip1]['meta'])
def test_41(self): """ Horizontal Interpolation See also: """ import os, sys, datetime import rpnpy.librmn.all as rmn fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate) fileNameOut = 'p0fstfile.fst' # Restric to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) try: # Create Destination grid # Note: Destination grid can also be read from a file gp = { 'grtyp' : 'Z', 'grref' : 'E', 'ni' : 90, 'nj' : 45, 'lat0' : 35., 'lon0' : 250., 'dlat' : 0.5, 'dlon' : 0.5, 'xlat1' : 0., 'xlon1' : 180., 'xlat2' : 1., 'xlon2' : 270. } gOut = rmn.encodeGrid(gp) print("CB41: Defined a %s/%s grid of shape=%d, %d" % (gOut['grtyp'], gOut['grref'], gOut['ni'], gOut['nj'])) except: sys.stderr.write("Problem creating grid\n") sys.exit(1) # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Find and read record to interpolate with its grid r = rmn.fstlir(fileIdIn, nomvar='P0') gIn = rmn.readGrid(fileIdIn, r) print("CB41: Read P0") # Set interpolation options and interpolate rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR) d = rmn.ezsint(gOut, gIn, r) print("CB41: Interpolate P0") # Create new record to write with interpolated data and r2 = r.copy() # Preserve meta from original record r2.update(gOut) # update grid information r2.update({ # attach data and update specific meta 'etiket': 'my_etk', 'd' : d }) # Write record data + meta + grid to file rmn.fstecr(fileIdOut, r2) rmn.writeGrid(fileIdOut, gOut) print("CB41: Wrote interpolated P0 and its grid") except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
except: sys.stderr.write("ERROR: Problem opening the file: {}\n". format(args.outputFile)) sys.exit(1) status = 0 for filename in args.inputFile: try: txtRec = txtFile2Rec(filename, args.verbose) fstRec = txt2fstRec(txtRec, args.verbose) if args.verbose: (yyyymmdd, hhmmss00) = rmn.newdate(rmn.NEWDATE_STAMP2PRINT, fstRec['dateo']) sys.stdout.write("Writing to File: {} {}.{}\n". format(fstRec['nomvar'], yyyymmdd, hhmmss00)) rmn.fstecr(fileIdOut, fstRec['d'], fstRec) except: sys.stderr.write("ERROR: Problem encountered ---- ABORT\n") rmn.fstcloseall(fileIdOut) status = 1 break rmn.fstcloseall(fileIdOut) sys.exit(status) # -*- Mode: C; tab-width: 4; indent-tabs-mode: nil -*- # vim: set expandtab ts=4 sw=4: # kate: space-indent on; indent-mode cstyle; indent-width 4; mixedindent off;
def test_23(self): """ Edit: Read, Edit, Write records with meta, grid and vgrid This example shows how to * select records in a RPNStd file * read the record data + meta * edit/use record data and meta (compute the wind velocity) * write the recod data + meta * copy (read/write) the record grid descriptors * copy (read/write) the file vgrid descriptor See also: rpnpy.librmn.fstd98.fstopt rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrinl rpnpy.librmn.fstd98.fsrluk rpnpy.librmn.fstd98.fsrlir rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.grids.readGrid rpnpy.librmn.grids.writeGrid rpnpy.vgd.base.vgd_read rpnpy.vgd.base.vgd_write rpnpy.librmn.const rpnpy.vgd.const """ import os, sys, datetime from scipy.constants import knot as KNOT2MS import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate) fileNameOut = 'uvfstfile.fst' # Restrict to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST) # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Copy the vgrid descriptor v = vgd.vgd_read(fileIdIn) vgd.vgd_write(v, fileIdOut) print("CB23: Copied the vgrid descriptor") # Loop over the list of UU records to copy uu = {'d': None} vv = {'d': None} uvarray = None copyGrid = True for k in rmn.fstinl(fileIdIn, nomvar='UU'): # Read the UU record data and meta from fileNameIn # Provide data array to re-use memory uu = rmn.fstluk(k, dataArray=uu['d']) # Read the corresponding VV # Provide data array to re-use memory vv = rmn.fstlir(fileIdIn, nomvar='VV', ip1=uu['ip1'], ip2=uu['ip2'], datev=uu['datev'], dataArray=vv['d']) # Compute the wind modulus in m/s # Copy metadata from the UU record # Create / re-use memory space for computation results uv = uu.copy() if uvarray is None: uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN') uv['d'] = uvarray uv['d'][:, :] = np.sqrt(uu['d']**2. + vv['d']**2.) uv['d'] *= KNOT2MS # Convert from knot to m/s # Set new record name and Write it to fileNameOut uv['nomvar'] = 'WSPD' rmn.fstecr(fileIdOut, uv) print("CB23: Wrote %s ip1=%d, ip2=%d, dateo=%s : mean=%f" % (uv['nomvar'], uv['ip1'], uv['ip2'], uv['dateo'], uv['d'].mean())) # Read and Write grid (only once, all rec are on the same grid) if copyGrid: copyGrid = False g = rmn.readGrid(fileIdIn, uu) rmn.writeGrid(fileIdOut, g) print("CB23: Copied the grid descriptors") except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def to_fst(valid_date, fst_template, args): #output data to std file import os import time import copy import logging import numpy as np import rpnpy.librmn.all as rmn from rpnpy.rpndate import RPNDate from domutils import radar_tools import domutils._py_tools as dpy logger = logging.getLogger(logging_basename) logger.info('to_fst starting to process date: ' + str(valid_date)) #output filename and directory output_file = args.output_dir + valid_date.strftime(args.fst_file_struc) #if in complete mode and file exists, return and test next one if (os.path.isfile(output_file) and args.complete_dataset): logger.info(output_file + ' exists and complete_dataset=True. Skipping to the next.') return np.array([1]) elif os.path.isfile(output_file): #file exists but we are not completing a dataset erase file before making a new one os.remove(output_file) else: #we will create or overwrite the file; before that make sure directory exists this_fst_dir = os.path.dirname(output_file) dpy.parallel_mkdir(this_fst_dir) #get destination grid from PR template dest_lon = fst_template['lon'] dest_lat = fst_template['lat'] #reading the data if args.accum_len is not None: desired_quantity = 'accumulation' fst_var_name = 'PR' dat_dict = radar_tools.get_accumulation( end_date=valid_date, duration=args.accum_len, desired_quantity=desired_quantity, data_path=args.radar_data_dir, odim_latlon_file=args.h5_latlon_file, data_recipe=args.h5_file_struc, dest_lon=dest_lon, dest_lat=dest_lat, median_filt=args.median_filt, smooth_radius=args.smooth_radius) if desired_quantity == 'accumulation': logger.warning( '!!!convert mm to m since PR quantity is outputted!!!') dat_dict['accumulation'] /= 1e3 data_quantity_name = desired_quantity data_date_name = 'end_date' else: # #if reflectivity is desired use: #desired_quantity='reflectivity' #fst_var_name='RDBZ' # #for precipitation rates used in LHN use: desired_quantity = 'precip_rate' fst_var_name = 'RDPR' #get, convert, interpolate and smooth ODIM Reflectivity mosaics dat_dict = radar_tools.get_instantaneous( valid_date=valid_date, desired_quantity=desired_quantity, data_path=args.radar_data_dir, odim_latlon_file=args.h5_latlon_file, data_recipe=args.h5_file_struc, dest_lon=dest_lon, dest_lat=dest_lat, median_filt=args.median_filt, smooth_radius=args.smooth_radius) data_quantity_name = desired_quantity data_date_name = 'valid_date' #if we got nothing, fill output with nodata and zeros if dat_dict is None: logger.warning( 'no data found or file unreadeable, I observations are set to -9999. with quality index = 0.' ) expected_shape = dest_lat.shape precip_rate = np.full(expected_shape, -9999.) quality_index = np.zeros(expected_shape) data_valid_date = valid_date else: precip_rate = dat_dict[data_quantity_name] quality_index = dat_dict['total_quality_index'] data_valid_date = dat_dict[data_date_name] #mettre etiket if args.median_filt is None: etiquette_median_filt = 0 else: etiquette_median_filt = args.median_filt if args.smooth_radius is None: etiquette_smooth_radius = 0 else: etiquette_smooth_radius = args.smooth_radius etiket = 'MED' + "{:1d}".format( etiquette_median_filt) + 'SM' + "{:02d}".format( etiquette_smooth_radius) #prepare std objects #cmc timestamp date_obj = RPNDate(valid_date) cmc_timestamp = date_obj.datev #open fst file logger.info('writing ' + output_file) iunit = rmn.fstopenall(output_file, rmn.FST_RW) #write >> ^^ to output file rmn.writeGrid(iunit, fst_template['grid']) #make RDPR entry rdpr_entry = copy.deepcopy(fst_template['meta']) rdpr_entry['nomvar'] = 'RDPR' rdpr_entry['etiket'] = etiket rdpr_entry['dateo'] = cmc_timestamp rdpr_entry['datev'] = cmc_timestamp rdpr_entry['ip2'] = 0 rdpr_entry['deet'] = 0 rdpr_entry['npas'] = 0 rdpr_entry['nbits'] = 32 rdpr_entry['typvar'] = 'I' rdpr_entry['d'] = np.asfortranarray(precip_rate, dtype='float32') #make RDQI entry rdqi_entry = copy.deepcopy(fst_template['meta']) rdqi_entry['nomvar'] = 'RDQI' rdqi_entry['etiket'] = etiket rdqi_entry['dateo'] = cmc_timestamp rdqi_entry['datev'] = cmc_timestamp rdqi_entry['ip2'] = 0 rdqi_entry['deet'] = 0 rdqi_entry['npas'] = 0 rdqi_entry['nbits'] = 32 rdqi_entry['typvar'] = 'I' rdqi_entry['d'] = np.asfortranarray(quality_index, dtype='float32') rmn.fstecr(iunit, rdpr_entry) rmn.fstecr(iunit, rdqi_entry) #close file rmn.fstcloseall(iunit) logger.info('Done writing ' + output_file) #make a figure for this std file if the argument figure_dir was provided if args.figure_dir is not None: radar_tools.plot_rdpr_rdqi(fst_file=output_file, this_date=valid_date, fig_dir=args.figure_dir, fig_format=args.figure_format, args=args) return np.array([1])
return FCDict def closeFST(fileID): ''' Close the FST to prevent file corruption and the like. ''' rmn.fstcloseall(fileID) fileID = getID(filePath) for s in species: keyList1 = getKeys(fileID, ip1, s, 1) keyList2 = getKeys(fileID, ip1, s, 2) #print('keylist1 len is {}'.format(len(keyList1))) #print('keyList2 len is {}'.format(len(keyList2))) FCDict = createAFCDict(fileID, keyList1, keyList2, s) dateoList = FCDict.keys() dateoList.sort() #print('date list is len {}'.format(len(dateoList))) for date in dateoList: sumFCDict = sumArrays(FCDict, date, 'vals') for bInd, bin in enumerate(bins): scaleDict = scaleArray(sumFCDict, date, 'vals', bin, s, binApp[bInd]) rmn.fstecr(fileID, scaleDict[date]['vals'], scaleDict[date]['meta']) # print('New records written for spc {} and date {}.'.format(scaleDict[date]['meta']['nomvar'], date)) #print(dateoList) closeFST(fileID)
def test_24(self): """ Edit: New file from scratch This example shows how to * Create a record meta and data from scratch * Create grid descriptors for the data * Create vgrid descriptor for the data * write the recod data + meta * write the grid descriptors * write the vgrid descriptor See also: rpnpy.librmn.fstd98.fstopt rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.fstd98.dtype_fst2numpy rpnpy.librmn.grids.encodeGrid rpnpy.librmn.grids.defGrid_ZE rpnpy.librmn.grids.writeGrid rpnpy.librmn.base.newdate rpnpy.vgd.base.vgd_new rpnpy.vgd.base.vgd_new_pres rpnpy.vgd.base.vgd_new_get rpnpy.vgd.base.vgd_write rpnpy.librmn.const rpnpy.vgd.const """ import os, sys import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd # Restrict to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) # Create Record grid gp = { 'grtyp' : 'Z', 'grref' : 'E', 'ni' : 90, 'nj' : 45, 'lat0' : 35., 'lon0' : 250., 'dlat' : 0.5, 'dlon' : 0.5, 'xlat1' : 0., 'xlon1' : 180., 'xlat2' : 1., 'xlon2' : 270. } g = rmn.encodeGrid(gp) print("CB24: Defined a %s/%s grid of shape=%d, %d" % (gp['grtyp'], gp['grref'], gp['ni'], gp['nj'])) # Create Record vgrid lvls = (500.,850.,1000.) v = vgd.vgd_new_pres(lvls) ip1list = vgd.vgd_get(v, 'VIPT') print("CB24: Defined a Pres vgrid with lvls=%s" % str(lvls)) # Create Record data + meta datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed'] npdtype = rmn.dtype_fst2numpy(datyp) rshape = (g['ni'], g['nj'], len(ip1list)) r = rmn.FST_RDE_META_DEFAULT.copy() # Copy default record meta r.update(g) # Update with grid info r.update({ # Update with specific meta and data array 'nomvar': 'MASK', 'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000), 'nk' : len(ip1list), 'ip1' : 0, # level, will be set later, level by level 'ip2' : 6, # Forecast of 6h 'deet' : 3600, # Timestep in sec 'npas' : 6, # Step number 'etiket': 'my_etk', 'nbits' : 32, # Keep full 32 bits precision for that field 'datyp' : datyp, # datyp (above) float_IEEE_compressed 'd' : np.empty(rshape, dtype=npdtype, order='FORTRAN') }) print("CB24: Defined a new record of shape=%d, %d" % (r['ni'], r['nj'])) # Compute record values r['d'][:,:,:] = 0. r['d'][10:-11,5:-6,:] = 1. # Open Files fileNameOut = 'newfromscratch.fst' try: fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the file: %s, %s\n" % fileNameOut) sys.exit(1) # Write record data + meta + grid + vgrid to file try: r2d = r.copy() r2d['nk'] = 1 for k in range(len(ip1list)): r2d['ip1'] = ip1list[k] r2d['d'] = np.asfortranarray(r['d'][:,:,k]) rmn.fstecr(fileIdOut, r2d['d'], r2d) print("CB24: wrote %s at ip1=%d" % (r2d['nomvar'], r2d['ip1'])) rmn.writeGrid(fileIdOut, g) print("CB24: wrote the grid descriptors") vgd.vgd_write(v, fileIdOut) print("CB24: wrote the vgrid descriptor") except: raise finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
def test_23(self): """ Edit: Read, Edit, Write records with meta, grid and vgrid This example shows how to * select records in a RPNStd file * read the record data + meta * edit/use record data and meta (compute the wind velocity) * write the recod data + meta * copy (read/write) the record grid descriptors * copy (read/write) the file vgrid descriptor See also: rpnpy.librmn.fstd98.fstopt rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrinl rpnpy.librmn.fstd98.fsrluk rpnpy.librmn.fstd98.fsrlir rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.grids.readGrid rpnpy.librmn.grids.writeGrid rpnpy.vgd.base.vgd_read rpnpy.vgd.base.vgd_write rpnpy.librmn.const rpnpy.vgd.const """ import os, sys, datetime from scipy.constants import knot as KNOT2MS import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd fdate = datetime.date.today().strftime('%Y%m%d') + '00_048' CMCGRIDF = os.getenv('CMCGRIDF').strip() fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate) fileNameOut = 'uvfstfile.fst' # Restrict to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST) # Open Files try: fileIdIn = rmn.fstopenall(fileNameIn) fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut)) sys.exit(1) try: # Copy the vgrid descriptor v = vgd.vgd_read(fileIdIn) vgd.vgd_write(v, fileIdOut) print("CB23: Copied the vgrid descriptor") # Loop over the list of UU records to copy uu = {'d': None} vv = {'d': None} uvarray = None copyGrid = True for k in rmn.fstinl(fileIdIn, nomvar='UU'): # Read the UU record data and meta from fileNameIn # Provide data array to re-use memory uu = rmn.fstluk(k, dataArray=uu['d']) # Read the corresponding VV # Provide data array to re-use memory vv = rmn.fstlir(fileIdIn, nomvar='VV', ip1=uu['ip1'], ip2=uu['ip2'], datev=uu['datev'], dataArray=vv['d']) # Compute the wind modulus in m/s # Copy metadata from the UU record # Create / re-use memory space for computation results uv = uu.copy() if uvarray is None: uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN') uv['d'] = uvarray uv['d'][:,:] = np.sqrt(uu['d']**2. + vv['d']**2.) uv['d'] *= KNOT2MS # Convert from knot to m/s # Set new record name and Write it to fileNameOut uv['nomvar'] = 'WSPD' rmn.fstecr(fileIdOut, uv) print("CB23: Wrote %s ip1=%d, ip2=%d, dateo=%s : mean=%f" % (uv['nomvar'], uv['ip1'], uv['ip2'], uv['dateo'], uv['d'].mean())) # Read and Write grid (only once, all rec are on the same grid) if copyGrid: copyGrid = False g = rmn.readGrid(fileIdIn, uu) rmn.writeGrid(fileIdOut, g) print("CB23: Copied the grid descriptors") except: pass finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdIn) rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file
'ig2': tic_record['ip2'], 'ig3': tic_record['ip3'], 'ig4': tic_record['ig4'], 'deet': int(86400 / 4), # timestep in secs 'ip1': ip1 }) tmp = array[rp1] tmp = np.asfortranarray(tmp) # data array is structured as tmp = monthly_mean[level] where monthly_mean is [level, lat, lon] # Updates with data array in the form (lon x lat) new_record.update({'d': tmp.astype(np.float32)}) print "Defined a new record with dimensions ({0}, {1})".format( new_record['ni'], new_record['nj']) rmn.fstecr( file_id, new_record ) # write the dictionary record to the file as a new record zonal = zon_list[t] tmp = zonal[rp1] tmp = np.asfortranarray(tmp) zonal_record = new_record zonal_record.update({ 'nomvar': 'ZO', 'd': tmp.astype(np.float32) }) print "Defined a zonal mean record with dimensions ({0}, {1})".format( new_record['ni'], new_record['nj']) rmn.fstecr( file_id, zonal_record
def test_24(self): """ Edit: New file from scratch This example shows how to * Create a record meta and data from scratch * Create grid descriptors for the data * Create vgrid descriptor for the data * write the recod data + meta * write the grid descriptors * write the vgrid descriptor See also: rpnpy.librmn.fstd98.fstopt rpnpy.librmn.fstd98.fstopenall rpnpy.librmn.fstd98.fstcloseall rpnpy.librmn.fstd98.fsrecr rpnpy.librmn.fstd98.dtype_fst2numpy rpnpy.librmn.grids.encodeGrid rpnpy.librmn.grids.defGrid_ZE rpnpy.librmn.grids.writeGrid rpnpy.librmn.base.newdate rpnpy.vgd.base.vgd_new rpnpy.vgd.base.vgd_new_pres rpnpy.vgd.base.vgd_new_get rpnpy.vgd.base.vgd_write rpnpy.librmn.const rpnpy.vgd.const """ import os, sys import numpy as np import rpnpy.librmn.all as rmn import rpnpy.vgd.all as vgd # Restrict to the minimum the number of messages printed by librmn rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST) # Create Record grid gp = { 'grtyp': 'Z', 'grref': 'E', 'ni': 90, 'nj': 45, 'lat0': 35., 'lon0': 250., 'dlat': 0.5, 'dlon': 0.5, 'xlat1': 0., 'xlon1': 180., 'xlat2': 1., 'xlon2': 270. } g = rmn.encodeGrid(gp) print("CB24: Defined a %s/%s grid of shape=%d, %d" % (gp['grtyp'], gp['grref'], gp['ni'], gp['nj'])) # Create Record vgrid lvls = (500., 850., 1000.) v = vgd.vgd_new_pres(lvls) ip1list = vgd.vgd_get(v, 'VIPT') print("CB24: Defined a Pres vgrid with lvls=%s" % str(lvls)) # Create Record data + meta datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed'] npdtype = rmn.dtype_fst2numpy(datyp) rshape = (g['ni'], g['nj'], len(ip1list)) r = rmn.FST_RDE_META_DEFAULT.copy() # Copy default record meta r.update(g) # Update with grid info r.update({ # Update with specific meta and data array 'nomvar': 'MASK', 'dateo': rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000), 'nk': len(ip1list), 'ip1': 0, # level, will be set later, level by level 'ip2': 6, # Forecast of 6h 'deet': 3600, # Timestep in sec 'npas': 6, # Step number 'etiket': 'my_etk', 'nbits': 32, # Keep full 32 bits precision for that field 'datyp': datyp, # datyp (above) float_IEEE_compressed 'd': np.empty(rshape, dtype=npdtype, order='FORTRAN') }) print("CB24: Defined a new record of shape=%d, %d" % (r['ni'], r['nj'])) # Compute record values r['d'][:, :, :] = 0. r['d'][10:-11, 5:-6, :] = 1. # Open Files fileNameOut = 'newfromscratch.fst' try: fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW) except: sys.stderr.write("Problem opening the file: %s, %s\n" % fileNameOut) sys.exit(1) # Write record data + meta + grid + vgrid to file try: r2d = r.copy() r2d['nk'] = 1 for k in range(len(ip1list)): r2d['ip1'] = ip1list[k] r2d['d'] = np.asfortranarray(r['d'][:, :, k]) rmn.fstecr(fileIdOut, r2d['d'], r2d) print("CB24: wrote %s at ip1=%d" % (r2d['nomvar'], r2d['ip1'])) rmn.writeGrid(fileIdOut, g) print("CB24: wrote the grid descriptors") vgd.vgd_write(v, fileIdOut) print("CB24: wrote the vgrid descriptor") except: raise finally: # Properly close files even if an error occured above # This is important when editing to avoid corrupted files rmn.fstcloseall(fileIdOut) os.unlink(fileNameOut) # Remove test file