def read_center_bulk(rootdir,center,norm,platform,cycle,obtype,channel):
'Read data for a center'
fname = '%s/work/%s/%s/%s.h5' % (rootdir,center,norm,platform.lower())
tmp = lutils.readHDF(fname,'df')
tmp = loi.select(tmp,cycles=cycle,obtypes=obtype,channels=channel)
tmp.reset_index(inplace=True)
tmp = tmp.groupby(['DATETIME'])[['TotImp']].agg('sum')
tmp.columns = [center]
return tmp
def read_centers():
'Read binned data and return a dictionary of dataframes, one key per center'
df = {}
for center in centers:
fbinned = '%s/work/%s/%s/%s.binned.h5' % (rootdir, center, norm,
platform.lower())
tmp = lutils.readHDF(fbinned, 'df')
tmp = loi.select(tmp, cycles=cycle, obtypes=obtype, channels=channel)
tmp.reset_index(inplace=True)
tmp = tmp.groupby(['DATETIME', 'LONGITUDE',
'LATITUDE'])[['TotImp']].agg('sum')
tmp.columns = [center]
df[center] = tmp
return df
def main():
parser = ArgumentParser(
description='Create time-average data given bulk statistics',
formatter_class=ArgumentDefaultsHelpFormatter)
parser.add_argument(
'-c',
'--center',
help='originating center',
type=str,
required=True,
choices=['EMC', 'GMAO', 'NRL', 'JMA_adj', 'JMA_ens', 'MET'])
parser.add_argument(
'-r',
'--rootdir',
help='root path to directory',
type=str,
default=
'/scratch3/NCEPDEV/stmp2/Rahul.Mahajan/test/Thomas.Auligne/FSOI',
required=False)
args = parser.parse_args()
rootdir = args.rootdir
center = args.center
fname = '%s/work/%s/bulk_stats.h5' % (rootdir, center)
df = lutils.readHDF(fname, 'df')
df = loi.accumBulkStats(df)
platforms = loi.OnePlatform()
df = loi.groupBulkStats(df, platforms)
df = loi.tavg(df, level='PLATFORM')
fpkl = '%s/work/%s/tavg_stats.pkl' % (rootdir, center)
if os.path.isfile(fpkl):
os.remove(fpkl)
lutils.pickle(fpkl, df)
sys.exit(0)
default='dry',
choices=['dry', 'moist'],
required=False)
parser.add_argument(
'--rootdir',
help='root path to directory',
type=str,
default='/scratch3/NCEPDEV/stmp2/Rahul.Mahajan/test/Thomas.Auligne/FSOI',
required=False)
args = parser.parse_args()
rootdir = args.rootdir
center = args.center
norm = args.norm
fname = '%s/work/%s/%s/bulk_stats.h5' % (rootdir, center, norm)
df = lutils.readHDF(fname, 'df')
platforms = df.index.get_level_values('PLATFORM').unique()
for platform in sorted(platforms):
print '%s' % platform
tmp = df.xs(platform, level='PLATFORM', drop_level=False)
obtypes = tmp.index.get_level_values('OBTYPE').unique()
print ' | ' + ' '.join(obtypes)
if 'Tb' in obtypes:
channels = tmp.index.get_level_values('CHANNEL').unique()
print ' | ' + ' '.join(map(str, sorted(channels)))
print
sys.exit(0)
fbinned = '%s/work/%s/%s/%s.binned.h5' % (rootdir, center, norm,
platform.lower())
if os.path.isfile(fbinned): os.remove(fbinned)
for adate in pd.date_range(bdate, edate, freq='%dH' % interval):
adatestr = adate.strftime('%Y%m%d%H')
fname = '%s/data/%s/%s.%s.%s.h5' % (rootdir, center, center, norm,
adatestr)
if not os.path.isfile(fname):
print '%s : %s does not exist, SKIPPING ...' % (adatestr, fname)
continue
df = lutils.readHDF(fname, 'df', where=search_str)
if platform != df.index.get_level_values('PLATFORM').unique():
print '%s does not exist in %s' % (platform, fname)
print 'ABORTING'
raise
df = loi.bin_df(df, dlat=dlat, dlon=dlon)
lutils.writeHDF(fbinned,
'df',
df,
complevel=1,
complib='zlib',
fletcher32=True)
parser.add_argument('--center',help='originating center',type=str,choices=['GMAO','NRL','MET','MeteoFr','JMA_adj','JMA_ens','EMC'],required=True)
parser.add_argument('--platform',help='platform to extract and bin',type=str,required=True)
parser.add_argument('--norm',help='metric norm',type=str,default='dry',choices=['dry','moist'],required=False)
parser.add_argument('--rootdir',help='root path to directory',type=str,default='/scratch3/NCEPDEV/stmp2/Rahul.Mahajan/test/Thomas.Auligne/FSOI',required=False)
args = parser.parse_args()
center = args.center
rootdir = args.rootdir
norm = args.norm
platform = args.platform
search_str = ['PLATFORM="%s"' % (platform)]
fbulk = '%s/work/%s/%s/bulk_stats.h5' % (rootdir,center,norm)
df = lutils.readHDF(fbulk,'df',where=search_str)
if platform not in df.index.get_level_values('PLATFORM').unique():
print '%s does not exist in %s' % (platform,fbulk)
print 'ABORTING'
raise
df.reset_index(inplace=True)
df.drop('PLATFORM',axis=1,inplace=True)
df = df.groupby(['DATETIME','OBTYPE','CHANNEL']).agg('sum')
fbinned = '%s/work/%s/%s/%s.h5' % (rootdir,center,norm,platform.lower())
if os.path.isfile(fbinned): os.remove(fbinned)
lutils.writeHDF(fbinned,'df',df,complevel=1,complib='zlib',fletcher32=True)
sys.exit(0)
rootdir = args.rootdir
norm = args.norm
cycle = args.cycle
channel = args.channel
savefig = args.savefigure
if cycle is None:
cycle = [0, 6, 12, 18]
cyclestr = ''.join('%02dZ' % c for c in cycle)
else:
cycle = sorted(list(set(args.cycle)))
cyclestr = ''.join('%02dZ' % c for c in cycle)
fbinned = '%s/work/%s/%s/%s.binned.h5' % (rootdir, center, norm,
platform.lower())
df = lutils.readHDF(fbinned, 'df')
# Select by cycles, obtypes and channels
df = loi.select(df, cycles=cycle, obtypes=obtype, channels=channel)
# Aggregate over observation types
tmp = df.reset_index()
tmp.drop(['PLATFORM'], axis=1, inplace=True)
names = ['DATETIME', 'LONGITUDE', 'LATITUDE']
df = tmp.groupby(names)['TotImp'].agg('sum')
cmap = plt.cm.get_cmap(name='coolwarm', lut=20)
# plot means
tmp = df.reset_index()
names = ['LONGITUDE', 'LATITUDE']