def main(): parser = ArgumentParser(description='Process EMC file', formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-i', '--input', help='Raw EMC file', type=str, required=True) parser.add_argument('-o', '--output', help='Processed EMC HDF file', type=str, required=True) parser.add_argument('-a', '--adate', help='analysis date to process', metavar='YYYYMMDDHH', required=True) args = parser.parse_args() fname = args.input fname_out = args.output adate = datetime.strptime(args.adate, '%Y%m%d%H') idate, nobscon, nobsoz, nobssat, npred, nens = emc.get_header(fname, endian='big') nobs = nobscon + nobsoz + nobssat obtype, platform, chan, lat, lon, lev, omf, oberr, imp = emc.get_data( fname, nobs, npred, nens, endian='big') obtype = (obtype.tostring()).replace('\x00', '')[:-1].split('|') platform = (platform.tostring()).replace('\x00', '')[:-1].split('|') bufr = [] for o in range(nobs): obtyp = ''.join(obtype[o]).strip() platf = ''.join(platform[o]).strip() if o < (nobscon + nobsoz): plat = get_platform_con(platf) if plat in 'UNKNOWN': print o + 1, plat, platf else: plat = get_platform_rad(platf) lon[o] = lon[o] if lon[o] >= 0.0 else lon[o] + 360.0 line = [ plat, obtyp, chan[o], lon[o], lat[o], lev[o], imp[o][0], omf[o], oberr[o] ] bufr.append(line) if bufr != []: df = loi.list_to_dataframe(adate, bufr) if os.path.isfile(fname_out): os.remove(fname_out) lutils.writeHDF(fname_out, 'df', df, complevel=1, complib='zlib', fletcher32=True) print 'Total obs = %d' % (nobs) sys.exit(0)
if data is None: continue nobs += 1 plat = data['platform'] channel = data['channel'] obtype = data['obtype'] lon = data['lon'] if data['lon'] >= 0.0 else data['lon'] + 360.0 lat = data['lat'] lev = data['lev'] imp = data['impact'] omf = data['omf'] oberr = data['oberr'] line = [plat,obtype,channel,lon,lat,lev,imp,omf,oberr] bufr.append(line) if bufr != []: df = loi.list_to_dataframe(adate,bufr) if os.path.isfile(fname_out): os.remove(fname_out) lutils.writeHDF(fname_out,'df',df,complevel=1,complib='zlib',fletcher32=True) print 'Total obs = %d' % (nobs) sys.exit(0) if __name__ == '__main__': main()
def main(): parser = ArgumentParser(description='Process GMAO data', formatter_class=ArgumentDefaultsHelpFormatter) parser.add_argument('-i', '--indir', help='path to ODS directory', type=str, required=True) parser.add_argument('-o', '--output', help='Processed GMAO HDF file', type=str, required=True) parser.add_argument('-a', '--adate', help='analysis date to process', metavar='YYYYMMDDHH', required=True) parser.add_argument('-n', '--norm', help='norm to process', type=str, default='dry', choices=['dry', 'moist'], required=False) args = parser.parse_args() datapth = args.indir fname_out = args.output adate = datetime.strptime(args.adate, '%Y%m%d%H') norm = args.norm if norm in ['dry']: norm = 'txe' elif norm in ['moist']: norm = 'twe' kx = kx_def() kt = kt_def() datadir = os.path.join(datapth, adate.strftime('Y%Y'), adate.strftime('M%m'), adate.strftime('D%d')) flist = get_files(datadir, adate) nobs = 0 bufr = [] for fname in flist: if norm not in fname: continue print 'processing %s' % fname platform = fname.split('.')[1].split('imp3_%s_' % norm)[-1].upper() fname = os.path.join(datadir, fname) ods = ODS(fname) ods = ods.read(only_good=True, platform=platform) ods.close() nobs += ods.nobs print 'platform = %s, nobs = %d' % (platform, ods.nobs) for o in range(ods.nobs): plat = kx[ods.kx[o]] if platform in ['CONV'] else platform obtype = kt[ods.kt[o]][0] channel = -999 if platform in ['CONV'] else np.int(ods.lev[o]) lon = ods.lon[o] if ods.lon[o] >= 0.0 else ods.lon[o] + 360.0 lat = ods.lat[o] if obtype == 'ps': lev = ods.obs[o] elif obtype == 'Tb': lev = -999. else: lev = ods.lev[o] imp = ods.xvec[o] omf = ods.omf[o] oberr = -999. # GMAO does not provide obs. error in the impact ODS files line = [plat, obtype, channel, lon, lat, lev, imp, omf, oberr] bufr.append(line) if bufr != []: df = loi.list_to_dataframe(adate, bufr) if os.path.isfile(fname_out): os.remove(fname_out) lutils.writeHDF(fname_out, 'df', df, complevel=1, complib='zlib', fletcher32=True) print 'Total obs used in %s = %d' % (adate.strftime('%Y%m%d%H'), nobs) sys.exit(0)