def test_23qd(self):
import os, sys, datetime
from scipy.constants import knot as KNOT2MS
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
fileNameOut = 'uvfstfile.fst'
fileIdIn = rmn.fstopenall(os.getenv('CMCGRIDF')+'/prog/regeta/'+fdate)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
vgd.vgd_write(vgd.vgd_read(fileIdIn), fileIdOut)
(uu, vv, uvarray, copyGrid) = ({'d': None}, {'d': None}, None, True)
for k in rmn.fstinl(fileIdIn, nomvar='UU'):
uu = rmn.fstluk(k, dataArray=uu['d'])
vv = rmn.fstlir(fileIdIn, nomvar='VV', ip1=uu['ip1'], ip2=uu['ip2'],
datev=uu['datev'],dataArray=vv['d'])
if uvarray is None:
uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN')
uv = uu.copy()
uv.update({'d':uvarray, 'nomvar': 'WSPD'})
uv['d'][:,:] = np.sqrt(uu['d']**2. + vv['d']**2.) * KNOT2MS
rmn.fstecr(fileIdOut, uv)
if copyGrid:
copyGrid = False
rmn.writeGrid(fileIdOut, rmn.readGrid(fileIdIn, uu))
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_23qd(self):
import os, sys, datetime
from scipy.constants import knot as KNOT2MS
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
fileNameOut = 'uvfstfile.fst'
fileIdIn = rmn.fstopenall(
os.getenv('CMCGRIDF') + '/prog/regeta/' + fdate)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
vgd.vgd_write(vgd.vgd_read(fileIdIn), fileIdOut)
(uu, vv, uvarray, copyGrid) = ({'d': None}, {'d': None}, None, True)
for k in rmn.fstinl(fileIdIn, nomvar='UU'):
uu = rmn.fstluk(k, dataArray=uu['d'])
vv = rmn.fstlir(fileIdIn,
nomvar='VV',
ip1=uu['ip1'],
ip2=uu['ip2'],
datev=uu['datev'],
dataArray=vv['d'])
if uvarray is None:
uvarray = np.empty(uu['d'].shape,
dtype=uu['d'].dtype,
order='FORTRAN')
uv = uu.copy()
uv.update({'d': uvarray, 'nomvar': 'WSPD'})
uv['d'][:, :] = np.sqrt(uu['d']**2. + vv['d']**2.) * KNOT2MS
rmn.fstecr(fileIdOut, uv)
if copyGrid:
copyGrid = False
rmn.writeGrid(fileIdOut, rmn.readGrid(fileIdIn, uu))
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_24qd(self):
import os, sys
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
g = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.)
lvls = (500.,850.,1000.)
v = vgd.vgd_new_pres(lvls)
ip1list = vgd.vgd_get(v, 'VIPT')
datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed']
npdtype = rmn.dtype_fst2numpy(datyp)
rshape = (g['ni'], g['nj'], len(ip1list))
r = rmn.FST_RDE_META_DEFAULT.copy()
r.update(g)
r.update({
'nomvar': 'MASK', 'nk' : len(ip1list),
'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000),
'ip2' : 6, 'deet' : 3600, 'npas' : 6,
'etiket': 'my_etk', 'datyp' : datyp,
'd' : np.empty(rshape, dtype=npdtype, order='FORTRAN')
})
r['d'][:,:,:] = 0.
r['d'][10:-11,5:-6,:] = 1.
fileNameOut = 'newfromscratch.fst'
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
r2d = r.copy()
for k in range(len(ip1list)):
r2d.update({'nk':1, 'ip1':ip1list[k], 'd':np.asfortranarray(r['d'][:,:,k])})
rmn.fstecr(fileIdOut, r2d['d'], r2d)
rmn.writeGrid(fileIdOut, g)
vgd.vgd_write(v, fileIdOut)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_24qd(self):
import os, sys
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
g = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.)
lvls = (500., 850., 1000.)
v = vgd.vgd_new_pres(lvls)
ip1list = vgd.vgd_get(v, 'VIPT')
datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed']
npdtype = rmn.dtype_fst2numpy(datyp)
rshape = (g['ni'], g['nj'], len(ip1list))
r = rmn.FST_RDE_META_DEFAULT.copy()
r.update(g)
r.update({
'nomvar':
'MASK',
'nk':
len(ip1list),
'dateo':
rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000),
'ip2':
6,
'deet':
3600,
'npas':
6,
'etiket':
'my_etk',
'datyp':
datyp,
'd':
np.empty(rshape, dtype=npdtype, order='FORTRAN')
})
r['d'][:, :, :] = 0.
r['d'][10:-11, 5:-6, :] = 1.
fileNameOut = 'newfromscratch.fst'
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
r2d = r.copy()
for k in range(len(ip1list)):
r2d.update({
'nk': 1,
'ip1': ip1list[k],
'd': np.asfortranarray(r['d'][:, :, k])
})
rmn.fstecr(fileIdOut, r2d['d'], r2d)
rmn.writeGrid(fileIdOut, g)
vgd.vgd_write(v, fileIdOut)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_writeGrid(self):
ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES')
file0 = os.path.join(ATM_MODEL_DFILES.strip(),'bcmk/geophy.fst')
funit = rmn.fstopenall(file0)
rec = rmn.fstlir(funit, nomvar='ME')
grid0 = rmn.readGrid(funit, rec)
rmn.fstcloseall(funit)
grid1 = rmn.defGrid_L(180,60,0.,180.,1.,0.5)
grid2 = rmn.defGrid_ZE(90,45,10.,11.,1.,0.5,0.,180.,1.,270.)
grid3 = rmn.defGrid_YY(31,5,0.,180.,1.,270.)
self.erase_testfile()
myfile = self.fname
funit = rmn.fstopenall(myfile, rmn.FST_RW)
rmn.fstecr(funit,rec['d'],rec)
rmn.writeGrid(funit, grid0)
rmn.writeGrid(funit, grid1)
rmn.writeGrid(funit, grid2)
rmn.writeGrid(funit, grid3)
rmn.fstcloseall(funit)
funit = rmn.fstopenall(myfile, rmn.FST_RO)
rec = rmn.fstlir(funit, nomvar='ME')
grid0b = rmn.readGrid(funit, rec)
rmn.fstcloseall(funit)
self.erase_testfile()
for k in grid0.keys():
if isinstance(grid0[k],np.ndarray):
ok = np.any(np.abs(grid0b[k]-grid0[k]) > self.epsilon)
self.assertFalse(ok, 'For k=%s, grid0b - grid0 = %s' % (k,str(np.abs(grid0b[k]-grid0[k]))))
else:
self.assertEqual(grid0b[k],grid0[k], 'For k=%s, expected:%s, got:%s' % (k, str(grid0[k]), str(grid0b[k])))
def build_fst(params, y_int, m_int):
'''
(dict) -> (int, dict)
builds the file as per the parameters defined in the params dict
returns the file_id
'''
# makes an empty .fst file
day = time.gmtime()
temp = ''
for x in day:
temp += str(x)
#new_nc = '/home/ords/aq/alh002/pyscripts/workdir/pv_files/TEST5.fst'
new_nc = '/home/ords/aq/alh002/pyscripts/workdir/pv_files/SHIFTED_POTVOR_file_{0}_{1}.fst'.format(
y_int + 2008, m_int + 1)
tmp = open(new_nc, 'w+')
tmp.close()
output_file = new_nc
try:
file_id = rmn.fnom(output_file)
open_fst = rmn.fstouv(file_id, rmn.FST_RW)
print(file_id, open_fst)
MACC_grid = rmn.encodeGrid(params)
print("Grids created.")
print 'Grid Shape:' + str(MACC_grid['shape'])
rmn.writeGrid(file_id, MACC_grid)
toc_record = vgd.vgd_new_pres(const_pressure,
ip1=MACC_grid['ig1'],
ip2=MACC_grid['ig2'])
vgd.vgd_write(toc_record, file_id)
return file_id, MACC_grid
except:
rmn.fstfrm(file_id)
rmn.fclos(file_id)
raise
def test_41qd(self):
import os, sys, datetime
import rpnpy.librmn.all as rmn
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameOut = 'p0fstfileqd.fst'
fileIdIn = rmn.fstopenall(os.getenv('CMCGRIDF')+'/prog/regeta/'+fdate)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
gOut = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.)
r = rmn.fstlir(fileIdIn, nomvar='P0')
gIn = rmn.readGrid(fileIdIn, r)
rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR)
d = rmn.ezsint(gOut, gIn, r)
r2 = r.copy()
r2.update(gOut)
r2.update({'etiket':'my_etk', 'd':d})
rmn.fstecr(fileIdOut, r2)
rmn.writeGrid(fileIdOut, gOut)
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_41qd(self):
import os, sys, datetime
import rpnpy.librmn.all as rmn
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameOut = 'p0fstfileqd.fst'
fileIdIn = rmn.fstopenall(
os.getenv('CMCGRIDF') + '/prog/regeta/' + fdate)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
gOut = rmn.defGrid_ZE(90, 45, 35., 250., 0.5, 0.5, 0., 180., 1., 270.)
r = rmn.fstlir(fileIdIn, nomvar='P0')
gIn = rmn.readGrid(fileIdIn, r)
rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR)
d = rmn.ezsint(gOut, gIn, r)
r2 = r.copy()
r2.update(gOut)
r2.update({'etiket': 'my_etk', 'd': d})
rmn.fstecr(fileIdOut, r2)
rmn.writeGrid(fileIdOut, gOut)
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_writeGrid(self):
ATM_MODEL_DFILES = os.getenv('ATM_MODEL_DFILES')
file0 = os.path.join(ATM_MODEL_DFILES.strip(), 'bcmk/geophy.fst')
funit = rmn.fstopenall(file0)
rec = rmn.fstlir(funit, nomvar='ME')
grid0 = rmn.readGrid(funit, rec)
rmn.fstcloseall(funit)
grid1 = rmn.defGrid_L(180, 60, 0., 180., 1., 0.5)
grid2 = rmn.defGrid_ZE(90, 45, 10., 11., 1., 0.5, 0., 180., 1., 270.)
grid3 = rmn.defGrid_YY(31, 5, 0., 180., 1., 270.)
self.erase_testfile()
myfile = self.fname
funit = rmn.fstopenall(myfile, rmn.FST_RW)
rmn.fstecr(funit, rec['d'], rec)
rmn.writeGrid(funit, grid0)
rmn.writeGrid(funit, grid1)
rmn.writeGrid(funit, grid2)
rmn.writeGrid(funit, grid3)
rmn.fstcloseall(funit)
funit = rmn.fstopenall(myfile, rmn.FST_RO)
rec = rmn.fstlir(funit, nomvar='ME')
grid0b = rmn.readGrid(funit, rec)
rmn.fstcloseall(funit)
self.erase_testfile()
for k in grid0.keys():
if isinstance(grid0[k], np.ndarray):
ok = np.any(np.abs(grid0b[k] - grid0[k]) > self.epsilon)
self.assertFalse(
ok, 'For k=%s, grid0b - grid0 = %s' %
(k, str(np.abs(grid0b[k] - grid0[k]))))
else:
self.assertEqual(
grid0b[k], grid0[k], 'For k=%s, expected:%s, got:%s' %
(k, str(grid0[k]), str(grid0b[k])))
def test_41(self):
"""
Horizontal Interpolation
See also:
"""
import os, sys, datetime
import rpnpy.librmn.all as rmn
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate)
fileNameOut = 'p0fstfile.fst'
# Restrict to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST)
try:
# Create Destination grid
# Note: Destination grid can also be read from a file
gp = {
'grtyp': 'Z',
'grref': 'E',
'ni': 90,
'nj': 45,
'lat0': 35.,
'lon0': 250.,
'dlat': 0.5,
'dlon': 0.5,
'xlat1': 0.,
'xlon1': 180.,
'xlat2': 1.,
'xlon2': 270.
}
gOut = rmn.encodeGrid(gp)
print("CB41: Defined a %s/%s grid of shape=%d, %d" %
(gOut['grtyp'], gOut['grref'], gOut['ni'], gOut['nj']))
except:
sys.stderr.write("Problem creating grid\n")
sys.exit(1)
# Open Files
try:
fileIdIn = rmn.fstopenall(fileNameIn)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the files: %s, %s\n" %
(fileNameIn, fileNameOut))
sys.exit(1)
try:
# Find and read record to interpolate with its grid
r = rmn.fstlir(fileIdIn, nomvar='P0')
gIn = rmn.readGrid(fileIdIn, r)
print("CB41: Read P0")
# Set interpolation options and interpolate
rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR)
d = rmn.ezsint(gOut, gIn, r)
print("CB41: Interpolate P0")
# Create new record to write with interpolated data and
r2 = r.copy() # Preserve meta from original record
r2.update(gOut) # update grid information
r2.update({ # attach data and update specific meta
'etiket': 'my_etk',
'd': d
})
# Write record data + meta + grid to file
rmn.fstecr(fileIdOut, r2)
rmn.writeGrid(fileIdOut, gOut)
print("CB41: Wrote interpolated P0 and its grid")
except:
pass
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_24(self):
"""
Edit: New file from scratch
This example shows how to
* Create a record meta and data from scratch
* Create grid descriptors for the data
* Create vgrid descriptor for the data
* write the recod data + meta
* write the grid descriptors
* write the vgrid descriptor
See also:
rpnpy.librmn.fstd98.fstopt
rpnpy.librmn.fstd98.fstopenall
rpnpy.librmn.fstd98.fstcloseall
rpnpy.librmn.fstd98.fsrecr
rpnpy.librmn.fstd98.dtype_fst2numpy
rpnpy.librmn.grids.encodeGrid
rpnpy.librmn.grids.defGrid_ZE
rpnpy.librmn.grids.writeGrid
rpnpy.librmn.base.newdate
rpnpy.vgd.base.vgd_new
rpnpy.vgd.base.vgd_new_pres
rpnpy.vgd.base.vgd_new_get
rpnpy.vgd.base.vgd_write
rpnpy.librmn.const
rpnpy.vgd.const
"""
import os, sys
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
# Restrict to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST)
# Create Record grid
gp = {
'grtyp': 'Z',
'grref': 'E',
'ni': 90,
'nj': 45,
'lat0': 35.,
'lon0': 250.,
'dlat': 0.5,
'dlon': 0.5,
'xlat1': 0.,
'xlon1': 180.,
'xlat2': 1.,
'xlon2': 270.
}
g = rmn.encodeGrid(gp)
print("CB24: Defined a %s/%s grid of shape=%d, %d" %
(gp['grtyp'], gp['grref'], gp['ni'], gp['nj']))
# Create Record vgrid
lvls = (500., 850., 1000.)
v = vgd.vgd_new_pres(lvls)
ip1list = vgd.vgd_get(v, 'VIPT')
print("CB24: Defined a Pres vgrid with lvls=%s" % str(lvls))
# Create Record data + meta
datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed']
npdtype = rmn.dtype_fst2numpy(datyp)
rshape = (g['ni'], g['nj'], len(ip1list))
r = rmn.FST_RDE_META_DEFAULT.copy() # Copy default record meta
r.update(g) # Update with grid info
r.update({ # Update with specific meta and data array
'nomvar':
'MASK',
'dateo':
rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000),
'nk':
len(ip1list),
'ip1':
0, # level, will be set later, level by level
'ip2':
6, # Forecast of 6h
'deet':
3600, # Timestep in sec
'npas':
6, # Step number
'etiket':
'my_etk',
'nbits':
32, # Keep full 32 bits precision for that field
'datyp':
datyp, # datyp (above) float_IEEE_compressed
'd':
np.empty(rshape, dtype=npdtype, order='FORTRAN')
})
print("CB24: Defined a new record of shape=%d, %d" %
(r['ni'], r['nj']))
# Compute record values
r['d'][:, :, :] = 0.
r['d'][10:-11, 5:-6, :] = 1.
# Open Files
fileNameOut = 'newfromscratch.fst'
try:
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the file: %s, %s\n" %
fileNameOut)
sys.exit(1)
# Write record data + meta + grid + vgrid to file
try:
r2d = r.copy()
r2d['nk'] = 1
for k in range(len(ip1list)):
r2d['ip1'] = ip1list[k]
r2d['d'] = np.asfortranarray(r['d'][:, :, k])
rmn.fstecr(fileIdOut, r2d['d'], r2d)
print("CB24: wrote %s at ip1=%d" % (r2d['nomvar'], r2d['ip1']))
rmn.writeGrid(fileIdOut, g)
print("CB24: wrote the grid descriptors")
vgd.vgd_write(v, fileIdOut)
print("CB24: wrote the vgrid descriptor")
except:
raise
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def test_23(self):
"""
Edit: Read, Edit, Write records with meta, grid and vgrid
This example shows how to
* select records in a RPNStd file
* read the record data + meta
* edit/use record data and meta (compute the wind velocity)
* write the recod data + meta
* copy (read/write) the record grid descriptors
* copy (read/write) the file vgrid descriptor
See also:
rpnpy.librmn.fstd98.fstopt
rpnpy.librmn.fstd98.fstopenall
rpnpy.librmn.fstd98.fstcloseall
rpnpy.librmn.fstd98.fsrinl
rpnpy.librmn.fstd98.fsrluk
rpnpy.librmn.fstd98.fsrlir
rpnpy.librmn.fstd98.fsrecr
rpnpy.librmn.grids.readGrid
rpnpy.librmn.grids.writeGrid
rpnpy.vgd.base.vgd_read
rpnpy.vgd.base.vgd_write
rpnpy.librmn.const
rpnpy.vgd.const
"""
import os, sys, datetime
from scipy.constants import knot as KNOT2MS
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate)
fileNameOut = 'uvfstfile.fst'
# Restrict to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL, rmn.FSTOPI_MSG_CATAST)
# Open Files
try:
fileIdIn = rmn.fstopenall(fileNameIn)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the files: %s, %s\n" %
(fileNameIn, fileNameOut))
sys.exit(1)
try:
# Copy the vgrid descriptor
v = vgd.vgd_read(fileIdIn)
vgd.vgd_write(v, fileIdOut)
print("CB23: Copied the vgrid descriptor")
# Loop over the list of UU records to copy
uu = {'d': None}
vv = {'d': None}
uvarray = None
copyGrid = True
for k in rmn.fstinl(fileIdIn, nomvar='UU'):
# Read the UU record data and meta from fileNameIn
# Provide data array to re-use memory
uu = rmn.fstluk(k, dataArray=uu['d'])
# Read the corresponding VV
# Provide data array to re-use memory
vv = rmn.fstlir(fileIdIn,
nomvar='VV',
ip1=uu['ip1'],
ip2=uu['ip2'],
datev=uu['datev'],
dataArray=vv['d'])
# Compute the wind modulus in m/s
# Copy metadata from the UU record
# Create / re-use memory space for computation results
uv = uu.copy()
if uvarray is None:
uvarray = np.empty(uu['d'].shape,
dtype=uu['d'].dtype,
order='FORTRAN')
uv['d'] = uvarray
uv['d'][:, :] = np.sqrt(uu['d']**2. + vv['d']**2.)
uv['d'] *= KNOT2MS # Convert from knot to m/s
# Set new record name and Write it to fileNameOut
uv['nomvar'] = 'WSPD'
rmn.fstecr(fileIdOut, uv)
print("CB23: Wrote %s ip1=%d, ip2=%d, dateo=%s : mean=%f" %
(uv['nomvar'], uv['ip1'], uv['ip2'], uv['dateo'],
uv['d'].mean()))
# Read and Write grid (only once, all rec are on the same grid)
if copyGrid:
copyGrid = False
g = rmn.readGrid(fileIdIn, uu)
rmn.writeGrid(fileIdOut, g)
print("CB23: Copied the grid descriptors")
except:
pass
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
xlat2 = latitudearraynp[(ni / 2) + 1, nj / 2]
xlon2 = longitudearraynp[(ni / 2) + 1, nj / 2] + 360.0
params = rmn.defGrid_ZEr(ni, nj, lat0, lon0, dlat, dlon, xlat1, xlon1, xlat2,
xlon2)
funit = rmn.fstopenall('test.rpn', rmn.FST_RW)
print(fieldnp)
#rmn.fstecr(funit,np.asfortranarray(fieldnp))
params0 = {
'grtyp': 'Z',
'grref': 'E',
'ni': ni,
'nj': nj,
'lat0': lat0,
'lon0': lon0,
'dlat': dlat,
'dlon': dlon,
'xlat1': xlat1,
'xlon1': xlon1,
'xlat2': xlat2,
'xlon2': xlon2,
}
params = rmn.encodeGrid(params0)
print params
rmn.writeGrid(funit, params)
rmn.fstcloseall(funit)
def test_23(self):
"""
Edit: Read, Edit, Write records with meta, grid and vgrid
This example shows how to
* select records in a RPNStd file
* read the record data + meta
* edit/use record data and meta (compute the wind velocity)
* write the recod data + meta
* copy (read/write) the record grid descriptors
* copy (read/write) the file vgrid descriptor
See also:
rpnpy.librmn.fstd98.fstopt
rpnpy.librmn.fstd98.fstopenall
rpnpy.librmn.fstd98.fstcloseall
rpnpy.librmn.fstd98.fsrinl
rpnpy.librmn.fstd98.fsrluk
rpnpy.librmn.fstd98.fsrlir
rpnpy.librmn.fstd98.fsrecr
rpnpy.librmn.grids.readGrid
rpnpy.librmn.grids.writeGrid
rpnpy.vgd.base.vgd_read
rpnpy.vgd.base.vgd_write
rpnpy.librmn.const
rpnpy.vgd.const
"""
import os, sys, datetime
from scipy.constants import knot as KNOT2MS
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate)
fileNameOut = 'uvfstfile.fst'
# Restrict to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
# Open Files
try:
fileIdIn = rmn.fstopenall(fileNameIn)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut))
sys.exit(1)
try:
# Copy the vgrid descriptor
v = vgd.vgd_read(fileIdIn)
vgd.vgd_write(v, fileIdOut)
print("CB23: Copied the vgrid descriptor")
# Loop over the list of UU records to copy
uu = {'d': None}
vv = {'d': None}
uvarray = None
copyGrid = True
for k in rmn.fstinl(fileIdIn, nomvar='UU'):
# Read the UU record data and meta from fileNameIn
# Provide data array to re-use memory
uu = rmn.fstluk(k, dataArray=uu['d'])
# Read the corresponding VV
# Provide data array to re-use memory
vv = rmn.fstlir(fileIdIn, nomvar='VV',
ip1=uu['ip1'], ip2=uu['ip2'], datev=uu['datev'],
dataArray=vv['d'])
# Compute the wind modulus in m/s
# Copy metadata from the UU record
# Create / re-use memory space for computation results
uv = uu.copy()
if uvarray is None:
uvarray = np.empty(uu['d'].shape, dtype=uu['d'].dtype, order='FORTRAN')
uv['d'] = uvarray
uv['d'][:,:] = np.sqrt(uu['d']**2. + vv['d']**2.)
uv['d'] *= KNOT2MS # Convert from knot to m/s
# Set new record name and Write it to fileNameOut
uv['nomvar'] = 'WSPD'
rmn.fstecr(fileIdOut, uv)
print("CB23: Wrote %s ip1=%d, ip2=%d, dateo=%s : mean=%f" %
(uv['nomvar'], uv['ip1'], uv['ip2'], uv['dateo'], uv['d'].mean()))
# Read and Write grid (only once, all rec are on the same grid)
if copyGrid:
copyGrid = False
g = rmn.readGrid(fileIdIn, uu)
rmn.writeGrid(fileIdOut, g)
print("CB23: Copied the grid descriptors")
except:
pass
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def to_fst(valid_date, fst_template, args):
#output data to std file
import os
import time
import copy
import logging
import numpy as np
import rpnpy.librmn.all as rmn
from rpnpy.rpndate import RPNDate
from domutils import radar_tools
import domutils._py_tools as dpy
logger = logging.getLogger(logging_basename)
logger.info('to_fst starting to process date: ' + str(valid_date))
#output filename and directory
output_file = args.output_dir + valid_date.strftime(args.fst_file_struc)
#if in complete mode and file exists, return and test next one
if (os.path.isfile(output_file) and args.complete_dataset):
logger.info(output_file +
' exists and complete_dataset=True. Skipping to the next.')
return np.array([1])
elif os.path.isfile(output_file):
#file exists but we are not completing a dataset erase file before making a new one
os.remove(output_file)
else:
#we will create or overwrite the file; before that make sure directory exists
this_fst_dir = os.path.dirname(output_file)
dpy.parallel_mkdir(this_fst_dir)
#get destination grid from PR template
dest_lon = fst_template['lon']
dest_lat = fst_template['lat']
#reading the data
if args.accum_len is not None:
desired_quantity = 'accumulation'
fst_var_name = 'PR'
dat_dict = radar_tools.get_accumulation(
end_date=valid_date,
duration=args.accum_len,
desired_quantity=desired_quantity,
data_path=args.radar_data_dir,
odim_latlon_file=args.h5_latlon_file,
data_recipe=args.h5_file_struc,
dest_lon=dest_lon,
dest_lat=dest_lat,
median_filt=args.median_filt,
smooth_radius=args.smooth_radius)
if desired_quantity == 'accumulation':
logger.warning(
'!!!convert mm to m since PR quantity is outputted!!!')
dat_dict['accumulation'] /= 1e3
data_quantity_name = desired_quantity
data_date_name = 'end_date'
else:
#
#if reflectivity is desired use:
#desired_quantity='reflectivity'
#fst_var_name='RDBZ'
#
#for precipitation rates used in LHN use:
desired_quantity = 'precip_rate'
fst_var_name = 'RDPR'
#get, convert, interpolate and smooth ODIM Reflectivity mosaics
dat_dict = radar_tools.get_instantaneous(
valid_date=valid_date,
desired_quantity=desired_quantity,
data_path=args.radar_data_dir,
odim_latlon_file=args.h5_latlon_file,
data_recipe=args.h5_file_struc,
dest_lon=dest_lon,
dest_lat=dest_lat,
median_filt=args.median_filt,
smooth_radius=args.smooth_radius)
data_quantity_name = desired_quantity
data_date_name = 'valid_date'
#if we got nothing, fill output with nodata and zeros
if dat_dict is None:
logger.warning(
'no data found or file unreadeable, I observations are set to -9999. with quality index = 0.'
)
expected_shape = dest_lat.shape
precip_rate = np.full(expected_shape, -9999.)
quality_index = np.zeros(expected_shape)
data_valid_date = valid_date
else:
precip_rate = dat_dict[data_quantity_name]
quality_index = dat_dict['total_quality_index']
data_valid_date = dat_dict[data_date_name]
#mettre etiket
if args.median_filt is None:
etiquette_median_filt = 0
else:
etiquette_median_filt = args.median_filt
if args.smooth_radius is None:
etiquette_smooth_radius = 0
else:
etiquette_smooth_radius = args.smooth_radius
etiket = 'MED' + "{:1d}".format(
etiquette_median_filt) + 'SM' + "{:02d}".format(
etiquette_smooth_radius)
#prepare std objects
#cmc timestamp
date_obj = RPNDate(valid_date)
cmc_timestamp = date_obj.datev
#open fst file
logger.info('writing ' + output_file)
iunit = rmn.fstopenall(output_file, rmn.FST_RW)
#write >> ^^ to output file
rmn.writeGrid(iunit, fst_template['grid'])
#make RDPR entry
rdpr_entry = copy.deepcopy(fst_template['meta'])
rdpr_entry['nomvar'] = 'RDPR'
rdpr_entry['etiket'] = etiket
rdpr_entry['dateo'] = cmc_timestamp
rdpr_entry['datev'] = cmc_timestamp
rdpr_entry['ip2'] = 0
rdpr_entry['deet'] = 0
rdpr_entry['npas'] = 0
rdpr_entry['nbits'] = 32
rdpr_entry['typvar'] = 'I'
rdpr_entry['d'] = np.asfortranarray(precip_rate, dtype='float32')
#make RDQI entry
rdqi_entry = copy.deepcopy(fst_template['meta'])
rdqi_entry['nomvar'] = 'RDQI'
rdqi_entry['etiket'] = etiket
rdqi_entry['dateo'] = cmc_timestamp
rdqi_entry['datev'] = cmc_timestamp
rdqi_entry['ip2'] = 0
rdqi_entry['deet'] = 0
rdqi_entry['npas'] = 0
rdqi_entry['nbits'] = 32
rdqi_entry['typvar'] = 'I'
rdqi_entry['d'] = np.asfortranarray(quality_index, dtype='float32')
rmn.fstecr(iunit, rdpr_entry)
rmn.fstecr(iunit, rdqi_entry)
#close file
rmn.fstcloseall(iunit)
logger.info('Done writing ' + output_file)
#make a figure for this std file if the argument figure_dir was provided
if args.figure_dir is not None:
radar_tools.plot_rdpr_rdqi(fst_file=output_file,
this_date=valid_date,
fig_dir=args.figure_dir,
fig_format=args.figure_format,
args=args)
return np.array([1])
def test_41(self):
"""
Horizontal Interpolation
See also:
"""
import os, sys, datetime
import rpnpy.librmn.all as rmn
fdate = datetime.date.today().strftime('%Y%m%d') + '00_048'
CMCGRIDF = os.getenv('CMCGRIDF').strip()
fileNameIn = os.path.join(CMCGRIDF, 'prog', 'regeta', fdate)
fileNameOut = 'p0fstfile.fst'
# Restric to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
try:
# Create Destination grid
# Note: Destination grid can also be read from a file
gp = {
'grtyp' : 'Z',
'grref' : 'E',
'ni' : 90,
'nj' : 45,
'lat0' : 35.,
'lon0' : 250.,
'dlat' : 0.5,
'dlon' : 0.5,
'xlat1' : 0.,
'xlon1' : 180.,
'xlat2' : 1.,
'xlon2' : 270.
}
gOut = rmn.encodeGrid(gp)
print("CB41: Defined a %s/%s grid of shape=%d, %d" %
(gOut['grtyp'], gOut['grref'], gOut['ni'], gOut['nj']))
except:
sys.stderr.write("Problem creating grid\n")
sys.exit(1)
# Open Files
try:
fileIdIn = rmn.fstopenall(fileNameIn)
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the files: %s, %s\n" % (fileNameIn, fileNameOut))
sys.exit(1)
try:
# Find and read record to interpolate with its grid
r = rmn.fstlir(fileIdIn, nomvar='P0')
gIn = rmn.readGrid(fileIdIn, r)
print("CB41: Read P0")
# Set interpolation options and interpolate
rmn.ezsetopt(rmn.EZ_OPT_INTERP_DEGREE, rmn.EZ_INTERP_LINEAR)
d = rmn.ezsint(gOut, gIn, r)
print("CB41: Interpolate P0")
# Create new record to write with interpolated data and
r2 = r.copy() # Preserve meta from original record
r2.update(gOut) # update grid information
r2.update({ # attach data and update specific meta
'etiket': 'my_etk',
'd' : d
})
# Write record data + meta + grid to file
rmn.fstecr(fileIdOut, r2)
rmn.writeGrid(fileIdOut, gOut)
print("CB41: Wrote interpolated P0 and its grid")
except:
pass
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdIn)
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
def plotFSTs(season=season, spcs=spcs, spcsFiles=spcsFiles, outputName = outputName, saveDir=saveDir):
# print minimum outputs
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
mInds = []
for m in season:
mInds += [monthList.index(m)]
if os.path.exists(saveDir) == False:
nu.makeDir(saveDir)
for spcInd, nomvar in enumerate(spcs):
try:
filename = os.path.join(saveDir, 'output_file_{0}_{1}.fst'.format(outputName, nomvar))
print('Creating and saving to {}'.format(filename))
tmp = open(filename, 'w+'); tmp.close()
output_file = filename
file_id = rmn.fnom(output_file)
open_fst = rmn.fstouv(file_id, rmn.FST_RW)
open_file = spcsFiles[spcInd]
print "Parameter: " + nomvar
seaSpcData = get_var(pyg.open(open_file), nomvar, mInds)
nc_lnsp = pyg.open(lnsp_file)
pressures = get_pressures(nc_lnsp, mInds)
timelen, levlen, latlen, lonlen = seaSpcData.shape
#NOTE: uncomment the following three lines to prep data for basemap use
#lonShiftSSData = shift_lon(seaSpcData)
#vertInterpSSData = vert_interp(pressures, lonShiftSSData)
#meanSSData = np.mean(vertInterpSSData, axis=0)
#NOTE: uncommment the following four liness to use for fst plotting
vertInterpSSData = vert_interp(pressures, seaSpcData)
meanSSData = np.mean(vertInterpSSData, axis=0) # temp
for lvl, ray in enumerate(meanSSData):
meanSSData[lvl] = np.flipud(ray)
scaleFac = scaleSpcs[allSpcs.index(nomvar)]
scaledSSData = meanSSData*scaleFac
#define grid for this file - note that the MACC grid in the file is
#defined for lons -180 to 180, but the python defGrid_L can't deal
#with that and defines the grid from 0 to 360 so will have to reorder
#the MACC fields a bit, or they end up 180 deg out of phase
# Also necessary to add one more longitude to wrap around
dlatlon = 360./lonlen # this is equal to the resolution of the grid
params0 = {
'grtyp' : 'Z',
'grref' : 'L',
'nj' : latlen,
'ni' : lonlen,
'lat0' : -90.,
'lon0' : 0.,
'dlat' : dlatlon,
'dlon' : dlatlon
}
MACC_grid= rmn.encodeGrid(params0)
print("Grids created.")
print 'Grid Shape:' + str(MACC_grid['shape'])
# copies the default record
new_record = rmn.FST_RDE_META_DEFAULT.copy()
tic_record = rmn.FST_RDE_META_DEFAULT.copy()
tac_record = rmn.FST_RDE_META_DEFAULT.copy()
try:
rmn.writeGrid(file_id, MACC_grid)
tac = rmn.fstinl(file_id, nomvar='>>')[0]
tic = rmn.fstinl(file_id, nomvar='^^')[0]
tic_record.update(rmn.fstprm(tic))
tac_record.update(rmn.fstprm(tac))
tic_record.update({'datyp' : rmn.FST_DATYP_LIST['float']})
tac_record.update({'datyp' : rmn.FST_DATYP_LIST['float']})
rmn.fsteff(tic)
rmn.fsteff(tac)
tic_record.update({'d': MACC_grid['ay']})
tac_record.update({'d': MACC_grid['ax']})
toc_record = vgd.vgd_new_pres(const_pressure, ip1=MACC_grid['ig1'], ip2=MACC_grid['ig2'])
rmn.fstecr(file_id, tic_record) # write the dictionary record to the file as a new record
rmn.fstecr(file_id, tac_record) # write the dictionary record to the file as a new record
vgd.vgd_write(toc_record, file_id)
except:
raise
for rp1 in xrange(len(const_pressure)): # writes a record for every level (as a different ip1)
try:
# converts rp1 into a ip1 with pressure kind
ip1 = rmn.convertIp(rmn.CONVIP_ENCODE, const_pressure[rp1], rmn.KIND_PRESSURE)
new_record.update(MACC_grid)
new_record.update({ # Update with specific meta
'nomvar': nomvar,
'typvar': 'C',
'etiket': 'MACCRean',
'ni' : MACC_grid['ni'],
'nj' : MACC_grid['nj'],
'ig1' : tic_record['ip1'],
'ig2' : tic_record['ip2'],
'ig3' : tic_record['ip3'],
'ig4' : tic_record['ig4'],
'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20120101, 0000000),
'deet' : 0, # Timestep in sec
'ip1' : ip1
})
#tmp_nparray = np.asfortranarray(monthly_mean[rp1])
tmp = scaledSSData[rp1]
tmp = np.transpose(tmp)
# data array is structured as tmp = monthly_mean[level] where monthly_mean is [level, lat, lon]
new_record.update({'d': tmp.astype(np.float32)}) # Updates with data array in the form (lon x lat)
print "Defined a new record with dimensions ({0}, {1})".format(new_record['ni'], new_record['nj'])
rmn.fstecr(file_id, new_record) # write the dictionary record to the file as a new record
except:
#rmn.closeall(file_id)
rmn.fstfrm(file_id)
rmn.fclos(file_id)
raise
rmn.fstfrm(file_id)
rmn.fclos(file_id)
print('{} complete~'.format(filename))
except:
rmn.fstfrm(file_id)
rmn.fclos(file_id)
raise
print('Finished plotting all FSTs. ')
def test_24(self):
"""
Edit: New file from scratch
This example shows how to
* Create a record meta and data from scratch
* Create grid descriptors for the data
* Create vgrid descriptor for the data
* write the recod data + meta
* write the grid descriptors
* write the vgrid descriptor
See also:
rpnpy.librmn.fstd98.fstopt
rpnpy.librmn.fstd98.fstopenall
rpnpy.librmn.fstd98.fstcloseall
rpnpy.librmn.fstd98.fsrecr
rpnpy.librmn.fstd98.dtype_fst2numpy
rpnpy.librmn.grids.encodeGrid
rpnpy.librmn.grids.defGrid_ZE
rpnpy.librmn.grids.writeGrid
rpnpy.librmn.base.newdate
rpnpy.vgd.base.vgd_new
rpnpy.vgd.base.vgd_new_pres
rpnpy.vgd.base.vgd_new_get
rpnpy.vgd.base.vgd_write
rpnpy.librmn.const
rpnpy.vgd.const
"""
import os, sys
import numpy as np
import rpnpy.librmn.all as rmn
import rpnpy.vgd.all as vgd
# Restrict to the minimum the number of messages printed by librmn
rmn.fstopt(rmn.FSTOP_MSGLVL,rmn.FSTOPI_MSG_CATAST)
# Create Record grid
gp = {
'grtyp' : 'Z',
'grref' : 'E',
'ni' : 90,
'nj' : 45,
'lat0' : 35.,
'lon0' : 250.,
'dlat' : 0.5,
'dlon' : 0.5,
'xlat1' : 0.,
'xlon1' : 180.,
'xlat2' : 1.,
'xlon2' : 270.
}
g = rmn.encodeGrid(gp)
print("CB24: Defined a %s/%s grid of shape=%d, %d" %
(gp['grtyp'], gp['grref'], gp['ni'], gp['nj']))
# Create Record vgrid
lvls = (500.,850.,1000.)
v = vgd.vgd_new_pres(lvls)
ip1list = vgd.vgd_get(v, 'VIPT')
print("CB24: Defined a Pres vgrid with lvls=%s" % str(lvls))
# Create Record data + meta
datyp = rmn.FST_DATYP_LIST['float_IEEE_compressed']
npdtype = rmn.dtype_fst2numpy(datyp)
rshape = (g['ni'], g['nj'], len(ip1list))
r = rmn.FST_RDE_META_DEFAULT.copy() # Copy default record meta
r.update(g) # Update with grid info
r.update({ # Update with specific meta and data array
'nomvar': 'MASK',
'dateo' : rmn.newdate(rmn.NEWDATE_PRINT2STAMP, 20160302, 1800000),
'nk' : len(ip1list),
'ip1' : 0, # level, will be set later, level by level
'ip2' : 6, # Forecast of 6h
'deet' : 3600, # Timestep in sec
'npas' : 6, # Step number
'etiket': 'my_etk',
'nbits' : 32, # Keep full 32 bits precision for that field
'datyp' : datyp, # datyp (above) float_IEEE_compressed
'd' : np.empty(rshape, dtype=npdtype, order='FORTRAN')
})
print("CB24: Defined a new record of shape=%d, %d" % (r['ni'], r['nj']))
# Compute record values
r['d'][:,:,:] = 0.
r['d'][10:-11,5:-6,:] = 1.
# Open Files
fileNameOut = 'newfromscratch.fst'
try:
fileIdOut = rmn.fstopenall(fileNameOut, rmn.FST_RW)
except:
sys.stderr.write("Problem opening the file: %s, %s\n" % fileNameOut)
sys.exit(1)
# Write record data + meta + grid + vgrid to file
try:
r2d = r.copy()
r2d['nk'] = 1
for k in range(len(ip1list)):
r2d['ip1'] = ip1list[k]
r2d['d'] = np.asfortranarray(r['d'][:,:,k])
rmn.fstecr(fileIdOut, r2d['d'], r2d)
print("CB24: wrote %s at ip1=%d" % (r2d['nomvar'], r2d['ip1']))
rmn.writeGrid(fileIdOut, g)
print("CB24: wrote the grid descriptors")
vgd.vgd_write(v, fileIdOut)
print("CB24: wrote the vgrid descriptor")
except:
raise
finally:
# Properly close files even if an error occured above
# This is important when editing to avoid corrupted files
rmn.fstcloseall(fileIdOut)
os.unlink(fileNameOut) # Remove test file
