def gather_information(directory, pattern, force=False, report=None, verbose=0):
"""
Read Information for NetCDF global attributes about time and position of GPS RO event
Parameters
----------
directory search directory for GPS RO NetCDF files
pattern search pattern
Returns
-------
pandas.DataFrame
Notes
-----
Can take quite long to collect (2h)
stores info in outdir/GPS_info.h5
"""
from ncio import read_history
from raso.config import outdir
# check midpoints of GPS profiles and create a list with
filename = outdir + '/GPS_info.h5'
if os.path.isfile(filename) and not force:
tmp = pd.read_hdf(filename, 'data')
journal('GPSRO_info : %s %s' % (filename, tmp.shape), report, verbose)
return tmp
# filenames , lon, lat, time
files = find_files(directory, pattern, recursive=True)
data = {}
pbar = ProgressBar(maxval=len(files))
pbar.start()
for i, ifile in enumerate(files):
hist = read_history(ifile)
data[i] = {'file': ifile, 'lon': hist['lon'],
'lat': hist['lat'],
'date': datetime(hist['year'], hist['month'], hist['day'], hist['hour'], hist['minute'])}
pbar.update(i + 1)
pbar.finish()
data = pd.DataFrame.from_dict(data, orient='index')
# save
data.to_hdf(filename, 'data', format='table', append=False)
journal('GPSRO_info : %s %s' % (filename, data.shape), report, verbose)
return data
def from_gruan(sonde, directory=None, save=True,
var=['cor_temp', 'temp', 'press', 'cor_rh', 'rh', 'WVMR', 'alt', 'u_temp', 'u_rh', 'u_press'],
force=False, attach=None, attach_var=None, verbose=0, **kwargs):
"""
Read GRUAN Data and convert to a radiosonde class object
Read and convert time to std_times, rename temp and press to t and p.
Drop duplicate entries.
Possible Station Names:
'BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU',
'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD', 'TAT'
Parameters
----------
sonde str Appreviation of Sonde Station
directory str Directory of GRUAN data
var list of variables to include
save bool Save radiosonde to store ?
force bool Force a raw data reread ?
attach radiosonde Attach to radiosonde class object
attach_var list Variables to attach from HDFStore
verbose int Level of verbosity
kwargs dict Additional Keywords
Returns
-------
radiosonde class object
Raises
------
ValueError sonde not in possible Stations
"""
from raso.standard_dates_times import _fix_datetime
from raso.config import outdir
from raso import radiosonde
from . import from_store
from ncio import read_netcdf, read_history
avail_sondes = ['BAR', 'BEL', 'BOU', 'CAB', 'LAU', 'LIN', 'MAN', 'NAU', 'NYA', 'PAY', 'POT', 'REU', 'SGP', 'SOD',
'TAT']
if not isinstance(sonde,str):
raise ValueError("Requires a str: sonde")
if attach is not None:
if not isinstance(attach,radiosonde):
raise ValueError("Requires a radiosonde class object: attach")
if sonde not in avail_sondes:
raise ValueError("Only one of %s allowed!" % str(avail_sondes))
filename = outdir + "/GRUAN_" + sonde + ".h5"
if os.path.isfile(filename) and not force:
print "Recovering: %s"%filename
isonde = from_store(0, filename=filename)
if attach is not None:
if attach_var is None:
attach_var = isonde.vars
for ivar in isonde.vars:
if ivar in attach_var:
attach.add_data(ivar, getattr(isonde,ivar), history="GRUAN, %s %s" %(sonde, ivar))
return
else:
return isonde
if directory is None:
raise RuntimeError("[GRUAN] requires a directory!")
#
print_verbose("[GRUAN] Find files ...", verbose)
files = find_files(directory, '*' + sonde + '*.nc', recursive=True)
print_verbose("[GRUAN][%s] Files found: %d" % (sonde, len(files)), verbose)
data = []
if verbose > 0:
pbar = ProgressBar(maxval=len(files))
pbar.start()
i = 0
for ifile in files:
tmp = pd.DataFrame(read_netcdf(ifile, verbose=verbose - 2)) # convert it all to a big Frame
# select only necessary information
tmp = tmp[ var ]
tmp.index.name = 'orig_date' # retain original sounding times
tmp.rename(columns=lambda x: x.replace('temp', 't').replace('press', 'p').replace('rh','r'), inplace=True) # rename
tmp.p *= 100. # from hPa to Pa
# WVMR to q
tmp['q'] = tmp.eval("WVMR/(WVMR+1)") # specific humidity
tmp['date'] = tmp.index.to_period(freq='h').to_datetime() # truncate to hours
tmp = tmp.reset_index().set_index('date') # change index
rx = map(_fix_datetime, tmp.index) # change time to 0, 6, 12, 18 UTC
tmp.index = rx
tmp.drop_duplicates(inplace=True)
data.append(tmp)
if verbose > 0:
pbar.update(i + 1)
i += 1
if verbose > 0:
pbar.finish()
hist = read_history(ifile) #
try:
ident = "%06d" % int(hist['g.General.SiteWmoId'])
except:
ident = sonde
lat = float(hist['g.MeasuringSystem.Latitude'].split(' ')[0])
lon = float(hist['g.MeasuringSystem.Longitude'].split(' ')[0])
alt = float(hist['g.MeasuringSystem.Altitude'].split(' ')[0])
#
print_verbose("[GRUAN] Concatenate data ...",verbose)
data = pd.concat(data, axis=0)
# drop duplicates
# print_verbose("[GRUAN] Drop duplicates ...", verbose)
# data.drop_duplicates(inplace=True)
#
if attach is None:
print_verbose("[GRUAN] Create radiosonde object ...", verbose)
isonde = radiosonde(ident)
isonde.filename = filename
isonde.add_attr('lon', lon)
isonde.add_attr('lat', lat)
isonde.add_attr('alt', alt)
isonde.add_attr('is_gruan',True)
isonde.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
isonde.save(filename=filename, verbose=verbose)
return isonde
else:
attach.add_data('gruan', data, history="GRUAN, %s " %(sonde))
if save:
attach.save(verbose=verbose)
return