def __init__(self, fmt, flIn, latlong, freq='d'):
self.fileIn = flIn
self._fmtIn = fmt
if fmt is 'csv':
self.data = pd.read_csv(flIn,
parse_dates={'Timestamp': ['Date']},
index_col='Timestamp')
if fmt is 'ts':
self.data = ts.tsfromtxt(flIn,
delimiter=',',
freq=freq,
skiprows=1,
datecols=2,
names=['ID', 'PARAM', 'Flow', 'SYM'])
self.idGage = self.data['ID'][0]
self._lat = latlong[0]
self._long = latlong[1]
self._latMIN = self._lat * 60.
self._lonMIN = self._long * 60.
if self.data.freqstr == 'D':
self._nhours = 24
elif self.data.freqstr == 'H':
self._nhours = 1
def get_nm_climate(station='kabq', start_date='20040101', end_date='today'):
if end_date == 'today':
end_date = localtime()
end_date = '%i%.2i%.2i' % (end_date.tm_year, end_date.tm_mon,
end_date.tm_mday)
url = 'http://weather.nmsu.edu/cgi-shl/cns/stat.pl?station=%s&type=daily' \
'&smonth=%s&sday=%s&syear=%s' \
'&emonth=%s&eday=%s&eyear=%s' % (station,
start_date[4:6],
start_date[6:],
start_date[2:4],
end_date[4:6],
end_date[6:],
end_date[2:4])
data = urlopen(url)
data = data.readlines()
csv_data = []
accum_precip = 0.0
for row in data[9:-5]:
row = row.split('|')
precip = float(row[19]) - accum_precip
if precip < 0:
precip = 0
accum_precip = 0
accum_precip += precip
csv_data.append(' '.join(
[row[0].strip(), row[1].strip(), row[3].strip(),
str(precip)]))
csv_data = '\n'.join(csv_data)
climate = tsfromtxt(StringIO(csv_data),
dtype='float',
datecols=0,
missing='-999',
dateconverter=parse_nmsu_date)
return climate
def test_tsfromtxt(self):
"Tests reading from a text file."
fcontent = """#
'Dates', 'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'2007-01', 'strings',1,1.0,'mixed column',,1
'2007-02', 'with embedded "double quotes"',2,2.0,1.0,,1
'2007-03', 'strings',3,3.0E5,3,,1
'2007-05','strings',4,-1e-10,,,1
"""
import os
from datetime import datetime
import tempfile
(tmp_fd, tmp_fl) = tempfile.mkstemp()
os.write(tmp_fd, fcontent)
os.close(tmp_fd)
mrectxt = tsfromtxt(tmp_fl, delimiter=',', names=tuple('ABCDEFG'),
datecols=0, skip_header=2, asrecarray=True)
os.remove(tmp_fl)
#
dlist = ['2007-%02i' % i for i in (1, 2, 3, 5)]
self.failUnless(isinstance(mrectxt, TimeSeriesRecords))
assert_equal(mrectxt._dates, date_array(dlist, 'M'))
assert_equal(mrectxt.dtype.names, ['A', 'B', 'C', 'D', 'E', 'F'])
assert_equal(mrectxt.F, [1, 1, 1, 1])
assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
assert_equal(mrectxt.C, [1, 2, 300000, -1e-10])
def test_tsfromtxt(self):
"Tests reading from a text file."
fcontent = """#
'Dates', 'One (S)','Two (I)','Three (F)','Four (M)','Five (-)','Six (C)'
'2007-01', 'strings',1,1.0,'mixed column',,1
'2007-02', 'with embedded "double quotes"',2,2.0,1.0,,1
'2007-03', 'strings',3,3.0E5,3,,1
'2007-05','strings',4,-1e-10,,,1
"""
import os
from datetime import datetime
import tempfile
(tmp_fd, tmp_fl) = tempfile.mkstemp()
os.write(tmp_fd, fcontent)
os.close(tmp_fd)
mrectxt = tsfromtxt(tmp_fl,
delimiter=',',
names=tuple('ABCDEFG'),
datecols=0,
skip_header=2,
asrecarray=True)
os.remove(tmp_fl)
#
dlist = ['2007-%02i' % i for i in (1, 2, 3, 5)]
self.failUnless(isinstance(mrectxt, TimeSeriesRecords))
assert_equal(mrectxt._dates, date_array(dlist, 'M'))
assert_equal(mrectxt.dtype.names, ['A', 'B', 'C', 'D', 'E', 'F'])
assert_equal(mrectxt.F, [1, 1, 1, 1])
assert_equal(mrectxt.E._mask, [1, 1, 1, 1])
assert_equal(mrectxt.C, [1, 2, 300000, -1e-10])
def parse_file(dat_file):
"""Return meta and timeseries objects from WEC dat file.
Input: opened WEC data file
Output: Dictionary of timeseries with meta-data attached
"""
meta = []
trigger = False
## Append lines to meta until you reach the data.
while not trigger:
line = dat_file.readline().strip()
meta.append(line)
if '***' in line:
dat_file.readline()
dat_file.readline()
trigger = True
## Create timeseries from dat_file
## Note: dat_file is at the correct seek() position from previous
## while loop
timeseries = tsfromtxt(fname=dat_file,delimiter=',',datecols=0, freq='T'
,dtype=float)
## Separate timeseries
ts_dict = separate_timeseries(timeseries)
## Create meta dictionary
meta_dict = parse_meta(meta)
## Mask bad values
for index in ts_dict.iterkeys():
for value in meta_dict['filters'].itervalues():
ts_dict[index] = numpy.ma.masked_values(ts_dict[index], value)
## Assign meta data to ts data
meta_ts_dict = assign_meta(ts_dict, meta_dict)
return meta_ts_dict
def parse_file(dat_file):
"""Return meta and timeseries objects from WEC dat file.
Input: opened WEC data file
Output: Dictionary of timeseries with meta-data attached
"""
meta = []
trigger = False
## Append lines to meta until you reach the data.
while not trigger:
line = dat_file.readline().strip()
meta.append(line)
if '***' in line:
dat_file.readline()
dat_file.readline()
trigger = True
## Create timeseries from dat_file
## Note: dat_file is at the correct seek() position from previous
## while loop
timeseries = tsfromtxt(fname=dat_file,
delimiter=',',
datecols=0,
freq='T',
dtype=float)
## Separate timeseries
ts_dict = separate_timeseries(timeseries)
## Create meta dictionary
meta_dict = parse_meta(meta)
## Mask bad values
for index in ts_dict.iterkeys():
for value in meta_dict['filters'].itervalues():
ts_dict[index] = numpy.ma.masked_values(ts_dict[index], value)
## Assign meta data to ts data
meta_ts_dict = assign_meta(ts_dict, meta_dict)
return meta_ts_dict
def load_jma(mode='COAPS', **options):
"""
Load the JMA 5-month running mean monthly SST anomalies over the Nino-3 zone
(5N-5S, 150W-90W) and returns a :class:`~scikits.hydroclimpy.enso.ENSOIndicator`
object.
Three different modes are available:
``coaps``
The data is downloaded from the
`COAPS website <ftp://www.coaps.fsu.edu/pub/JMA_SST_Index/jmasst1868-today.filter-5>`_.
``standard``
The data is downloaded from the JMA site and corresponds to the SST
anomalies computed using a
`fixed reference <http://ds.data.jma.go.jp/tcc/tcc/products/elnino/index/sstindex/base_period_7100/Nino_3/5rmean>`_
period (1971-2000).
``sliding``
The data is also downloaded from the JMA site, but corresponds this time
to anomalies computed using a
`sliding 30-year rule <http://ds.data.jma.go.jp/tcc/tcc/products/elnino/index/sstindex/sliding_30year_period/Nino_3/5rmean>`_.
By default, the configuration options are read from the configuration file.
These options can be overwritten by the optional input parameters.
Parameters
----------
mode : {'coaps','standard','sliding'}, optional
Mode describing the data to download.
options : dictionary
Optional parameters to parse to the ENSOIndicator for the definition of
ENSO indices.
thresholds : tuple
Low and high temperature thresholds for the definition of El Niño or
La Niña conditions.
By default, the JMA uses -0.5oC and +0.5oC.
minimum_size : int
Minimum number of consecutive months in El Niño/La Niña conditions
required for the definition of an episode.
By default, the JMA use 6 consecutive months.
reference_season : string, tuple
Months that must be in an episode for it to be valid.
By default, the original JMA COAPS index uses ``'OND'`` (October to
December included), while the newer COAPS index uses ``'NDJ'`` (November
to January included).
full_year : boolean
Whether episodes are defined from Oct. 1st to the following Sep. 30th,
or last only until the SST anomalies conditions are no longer met.
The original JMA COAPS index uses ``full_year=True``, while the newer
index uses ``full_year=False``.
"""
#
ensoarchive = dict(config.items('ENSO'))['ensoarchive']
if ensoarchive[-4:].lower() != '.zip':
ensoarchive += '.zip'
# Check the mode and load the proper options
modes = ('coaps', 'standard', 'sliding')
mode = mode.lower()
if mode not in modes:
raise ValueError("Invalid '%s' mode for the JMA indices: "\
"it should be in %s" % (mode, modes))
if mode == 'coaps':
cfg = dict(config.items('ENSO.JMA.COAPS'))
elif mode == 'standard':
cfg = dict(config.items('ENSO.JMA.Standard'))
elif mode == 'sliding':
cfg = dict(config.items('ENSO.JMA.Sliding'))
else:
raise ValueError("Invalid '%s' mode for the JMA indices: "\
"it should be in %s" % (mode, modes))
cfg.update(options)
#
datadir = cfg['datadir']
netfile = cfg['netfile']
archive = cfg['archive']
scalefactor = float(cfg.get('scalefactor', 1.))
#
try:
zipf = zipfile.ZipFile(ensoarchive, 'r')
ensoi = cPickle.loads(zipf.read(archive))
errmsg = "... Loading from existing archived file '%s'" % ensoarchive
ensologger.info(errmsg)
except (IOError, ValueError):
zipf = zipfile.ZipFile(ensoarchive, 'w')
ensologger.info("... Creating archive")
except KeyError:
zipf = zipfile.ZipFile(ensoarchive, 'a')
ensologger.info("... Appending to archive")
else:
if isinstance(ensoi, enso.ENSOIndicator):
ensoi.set_indices()
return ensoi
#
sourcedir = np.lib._datasource.DataSource(datadir)
dfile = sourcedir.open(netfile)
#
tmp = ts.tsfromtxt(
dfile,
delimiter=None,
dtype=float,
datecols=0,
freq='A',
skiprows=int(cfg.get('skipline', 0)),
missing=cfg.get('nodata', '999'),
)
if tmp.varshape != (12, ):
msg = "Unrecognized shape: it should have been (x, 12)."
raise TypeError(msg)
ensoi = enso.ENSOIndicator(tmp._series.ravel(),
start_date=tmp._dates[0].asfreq('M', 'START'))
ensoi *= scalefactor
_set_ensoindicator_options(ensoi, **cfg)
ensoi.set_indices()
#
# Store in the archive
ensologger.info("... Saving to archive")
zipf.writestr(archive, cPickle.dumps(ensoi))
zipf.close()
return ensoi
def load_jma(mode='COAPS', **options):
"""
Load the JMA 5-month running mean monthly SST anomalies over the Nino-3 zone
(5N-5S, 150W-90W) and returns a :class:`~scikits.hydroclimpy.enso.ENSOIndicator`
object.
Three different modes are available:
``coaps``
The data is downloaded from the
`COAPS website <ftp://www.coaps.fsu.edu/pub/JMA_SST_Index/jmasst1868-today.filter-5>`_.
``standard``
The data is downloaded from the JMA site and corresponds to the SST
anomalies computed using a
`fixed reference <http://ds.data.jma.go.jp/tcc/tcc/products/elnino/index/sstindex/base_period_7100/Nino_3/5rmean>`_
period (1971-2000).
``sliding``
The data is also downloaded from the JMA site, but corresponds this time
to anomalies computed using a
`sliding 30-year rule <http://ds.data.jma.go.jp/tcc/tcc/products/elnino/index/sstindex/sliding_30year_period/Nino_3/5rmean>`_.
By default, the configuration options are read from the configuration file.
These options can be overwritten by the optional input parameters.
Parameters
----------
mode : {'coaps','standard','sliding'}, optional
Mode describing the data to download.
options : dictionary
Optional parameters to parse to the ENSOIndicator for the definition of
ENSO indices.
thresholds : tuple
Low and high temperature thresholds for the definition of El Niño or
La Niña conditions.
By default, the JMA uses -0.5oC and +0.5oC.
minimum_size : int
Minimum number of consecutive months in El Niño/La Niña conditions
required for the definition of an episode.
By default, the JMA use 6 consecutive months.
reference_season : string, tuple
Months that must be in an episode for it to be valid.
By default, the original JMA COAPS index uses ``'OND'`` (October to
December included), while the newer COAPS index uses ``'NDJ'`` (November
to January included).
full_year : boolean
Whether episodes are defined from Oct. 1st to the following Sep. 30th,
or last only until the SST anomalies conditions are no longer met.
The original JMA COAPS index uses ``full_year=True``, while the newer
index uses ``full_year=False``.
"""
#
ensoarchive = dict(config.items('ENSO'))['ensoarchive']
if ensoarchive[-4:].lower() != '.zip':
ensoarchive += '.zip'
# Check the mode and load the proper options
modes = ('coaps', 'standard', 'sliding')
mode = mode.lower()
if mode not in modes:
raise ValueError("Invalid '%s' mode for the JMA indices: "\
"it should be in %s" % (mode, modes))
if mode == 'coaps':
cfg = dict(config.items('ENSO.JMA.COAPS'))
elif mode == 'standard':
cfg = dict(config.items('ENSO.JMA.Standard'))
elif mode == 'sliding':
cfg = dict(config.items('ENSO.JMA.Sliding'))
else:
raise ValueError("Invalid '%s' mode for the JMA indices: "\
"it should be in %s" % (mode, modes))
cfg.update(options)
#
datadir = cfg['datadir']
netfile = cfg['netfile']
archive = cfg['archive']
scalefactor = float(cfg.get('scalefactor', 1.))
#
try:
zipf = zipfile.ZipFile(ensoarchive, 'r')
ensoi = cPickle.loads(zipf.read(archive))
errmsg = "... Loading from existing archived file '%s'" % ensoarchive
ensologger.info(errmsg)
except (IOError, ValueError):
zipf = zipfile.ZipFile(ensoarchive, 'w')
ensologger.info("... Creating archive")
except KeyError:
zipf = zipfile.ZipFile(ensoarchive, 'a')
ensologger.info("... Appending to archive")
else:
if isinstance(ensoi, enso.ENSOIndicator):
ensoi.set_indices()
return ensoi
#
sourcedir = np.lib._datasource.DataSource(datadir)
dfile = sourcedir.open(netfile)
#
tmp = ts.tsfromtxt(dfile, delimiter=None, dtype=float, datecols=0, freq='A',
skiprows=int(cfg.get('skipline', 0)),
missing=cfg.get('nodata', '999'),)
if tmp.varshape != (12,):
msg = "Unrecognized shape: it should have been (x, 12)."
raise TypeError(msg)
ensoi = enso.ENSOIndicator(tmp._series.ravel(),
start_date=tmp._dates[0].asfreq('M', 'START'))
ensoi *= scalefactor
_set_ensoindicator_options(ensoi, **cfg)
ensoi.set_indices()
#
# Store in the archive
ensologger.info("... Saving to archive")
zipf.writestr(archive, cPickle.dumps(ensoi))
zipf.close()
return ensoi
def load_coaps_stationdata(station, cfgdict=coaps_config):
"""
Loads the temperatures and rainfall data for the current COAPS station.
Parameters
----------
station : int
COAPS station identification code.
cfgdict : dictionary
Dictionary of configuration options.
By default, the configuration options are read from the :file:`.climpyrc`
file
Returns
-------
stationdata : ClimateRecords
A :class:`~scikits.climpy.core.ClimateRecords` object, with fields:
tobs : int
Number of temperature observations for the day.
tmin : float
Minimum recorded temperature (oC).
tmax : float
Maximum recorded temperature (oC).
rain : float
Recorded amount of precipitation (mm).
"""
#
datadir = cfgdict['datadir']
coapsarchive = _check_archive(cfgdict)
#
archfilename = "COAPS%s" % station
try:
zipf = zipfile.ZipFile(coapsarchive, 'r')
stationdata = cPickle.loads(zipf.read(archfilename))
except IOError:
zipf = zipfile.ZipFile(coapsarchive, 'w')
except (KeyError, ValueError):
zipf = zipfile.ZipFile(coapsarchive, 'a')
else:
return stationdata
#
netfile = os.path.join(cfgdict['netfile.dir'], "%sascii.txt" % station)
sourcedir = np.lib._datasource.DataSource(datadir)
dfile = sourcedir.open(netfile)
skip = int(cfgdict['headerlines'])
missing_values = cfgdict['nodata']
dateconverter = lambda y, m, d : ts.Date('D',
year=int(y), month=int(m),
day=int(d))
converters = {4 : lambda s: np.round((float(s) - 32) / 1.8, 1),
5 : lambda s: np.round((float(s) - 32) / 1.8, 1),
6 : lambda s: 0.254 * float(s)}
ndtype = [('nobs', int), ('tmin', float), ('tmax', float), ('rain', float)]
series = ts.tsfromtxt(dfile, delimiter="\t", skiprows=skip,
missing=missing_values, converters=converters,
datecols=(0, 1, 2), dateconverter=dateconverter)
series = series.view(ndtype)
# Store metainfo
metainfo = []
dfile.seek(0)
for i in range(skip - 2):
row = dfile.readline().strip()
if not len(row):
continue
info = tuple([_.strip() for _ in row.split(":")])
if len(info) == 1:
info = (info, '')
metainfo.append(info)
series._optinfo.update(metainfo=dict(metainfo))
zipf.writestr(archfilename, cPickle.dumps(series))
zipf.close()
return series
