def test_force_reference(self):
mseries = ts.time_series(np.arange(24),
start_date=ts.Date('M','2001-01'))
aseries = ts.time_series([1,2,3], start_date=ts.Date('A', '2001-01'))
#
mtest = force_reference(aseries, mseries)
assert_equal(mtest.freq, ts.check_freq('M'))
assert_equal(mtest.dates[[0,-1]], mseries.dates[[0,-1]])
assert_equal(mtest, [1]*12+[2]*12)
mtest = force_reference(aseries, mseries, ma.sum)
assert_equal(mtest, [1]*12+[2]*12)
#
atest = force_reference(mseries, aseries)
assert_equal(atest.freq, ts.check_freq('A'))
assert_equal(atest.dates[[0,-1]], aseries.dates[[0,-1]])
assert_equal(atest, ma.array([5.5, 17.5, 0], mask=[0,0,1]))
atest = force_reference(mseries, aseries, ma.sum)
assert_equal(atest, ma.array([66, 210, 0], mask=[0,0,1]))
def test_force_reference(self):
mseries = ts.time_series(np.arange(24),
start_date=ts.Date('M', '2001-01'))
aseries = ts.time_series([1, 2, 3], start_date=ts.Date('A', '2001-01'))
#
mtest = force_reference(aseries, mseries)
assert_equal(mtest.freq, ts.check_freq('M'))
assert_equal(mtest.dates[[0, -1]], mseries.dates[[0, -1]])
assert_equal(mtest, [1] * 12 + [2] * 12)
mtest = force_reference(aseries, mseries, ma.sum)
assert_equal(mtest, [1] * 12 + [2] * 12)
#
atest = force_reference(mseries, aseries)
assert_equal(atest.freq, ts.check_freq('A'))
assert_equal(atest.dates[[0, -1]], aseries.dates[[0, -1]])
assert_equal(atest, ma.array([5.5, 17.5, 0], mask=[0, 0, 1]))
atest = force_reference(mseries, aseries, ma.sum)
assert_equal(atest, ma.array([66, 210, 0], mask=[0, 0, 1]))
def create_hdf5file(filename, freq, filedescr=""):
"""Creates a new hdf5file for impact analysis."""
# Open the file for appending ...............
h5file = openFile(filename, mode="a", title=filedescr)
# Create the main groups ....................
for (name, descr) in zip(("values", "differences", "apxpvalues"),
("Rainfall values (mm/d)",
"Rainfall differences (mm/d)",
"Approximate p-values (mm/d)",)):
group = h5file.createGroup(h5file.root, name, descr)
# Create the subgroups ......................
for name in ("values", "differences", "apxpvalues"):
h5file.createGroup("/%s" % name, 'modified', "Modified ENSO index")
h5file.createGroup("/%s" % name, 'standard', "Standard ENSO index")
h5file.createGroup("/%s" % name, 'noenso', "Comparison Modified-Standard ENSO indices")
# Check the frequency .......................
freq = check_freq(freq)
if freq == _c.FR_MTH:
description = MonthlyResults
elif freq in (_c.FR_QTREFEB, _c.FR_QTRSFEB, _c.FR_QTREMAY, _c.FR_QTRSMAY,
_c.FR_QTREAUG, _c.FR_QTRSAUG, _c.FR_QTRENOV, _c.FR_QTRSNOV):
description = DJFResults
elif freq in (_c.FR_QTREMAR, _c.FR_QTRSMAR, _c.FR_QTREJUN, _c.FR_QTRSJUN,
_c.FR_QTRESEP, _c.FR_QTRSSEP, _c.FR_QTREDEC, _c.FR_QTRSDEC,
_c.FR_QTR):
description = JFMResults
elif freq in (_c.FR_QTREJAN, _c.FR_QTRSJAN, _c.FR_QTREAPR, _c.FR_QTRSAPR,
_c.FR_QTREOCT, _c.FR_QTRSOCT, _c.FR_QTREOCT, _c.FR_QTRSOCT):
description = NDJResults
# Create the table for the 'values' group ...
for subgroup in ("modified", "standard"):
for (phase, label) in zip(("neutral", "warm", "cold", "noenso"),
("Neutral", "El Nino", "La Nina", "Global")):
tablename = "/values/%s" % subgroup
table = h5file.createTable(tablename, phase, description, label)
# Create the tables for the differences/apxpvalues
for group in ("differences", "apxpvalues"):
# Comparison between phases
for subgroup in ("modified", "standard"):
for (phase, label) in zip(("CN", "WN", "CW", "CG", "NG", "WG"),
("La Nina - Neutral",
"El Nino - Neutral",
"La Nina - El Nino",
"La Nina - Global",
"Neutral - Global",
"El Nino - Global"),):
table = h5file.createTable("/".join(['', group, subgroup]),
phase, description, label)
# Comparison between methods
for (phase, label) in zip(("cold", "warm", "neutral",),
("La Nina - (mod-std)",
"El Nino - (mod-std)",
"Neutral - (mod-std)",),):
table = h5file.createTable("/".join(['', group, 'noenso']),
phase, description, label)
return h5file
def load_coaps_period_networkdata(field, freq=None, func=None,
start_date=None, end_date=None,
cfgdict=coaps_config):
"""
Load data converted the given period for all the stations
Parameters
----------
field : str
Type of data to select.
Must be one of ('tmin', 'tmax', 'rain')
freq : var, optional
Period to convert the dataset to.
func : function, optional
Function with which to convert the dataset.
The function must output a 1D dataset.
start_date : var, optional
Starting date of the dataset.
end_date : var, optional
Ending date of the dataset.
Returns
-------
period_networkdata
Structured array of the converted data for all stations of the network.
The output dtype is ``[(station_id, float)]`` for all the station ids.
"""
# Make sure we have a valid data type
valid_field = ('tmin', 'tmax', 'rain')
if field not in valid_field:
errmsg = "Invalid datatype: should be in %s.\n(got '%s')"
raise ValueError(errmsg % (valid_field, field))
# Check the frequency
freq = ts.check_freq(freq)
# Load the dictionary of adjusted data
datadict = load_coaps_adjusted_networkdata(start_date=start_date,
end_date=end_date,
cfgdict=cfgdict)
# Define the list of station ids
coaps_ids = datadict.keys()
# Define the output dtype
ndtype = [("%s" % _, float) for _ in sorted(coaps_ids)]
# Choose one series as reference and convert it
reference = datadict[coaps_ids[0]]
reference = reference[field].convert(freq, func=func)
# Exit if we don't have a 1D series
if reference.ndim != 1:
errmsg = "Conversion error: the output dataset should be 1D.\n"\
"(got a %iD series instead)"
raise TypeError(errmsg % reference.ndim)
series = ts.time_series(np.empty(len(reference), dtype=ndtype),
dates=reference.dates)
series_values = series.series
for (id_, data) in datadict.items():
series_values[id_] = data[field].convert(freq, func=func).series
return series
def deseasonalize(series, season, normalize=True, unbiased=True):
"""
Deseasonalize a time series by substracting the seasonal average and
dividing by the seasonal standard deviation (if needed).
Returns a new time series, with the same frequency as the original.
Parameters
----------
series : TimeSeries
Input time series.
season : string/integer
Frequency of the season.
For example, use 'M' or 3000 for 12 seasons per year (one per month).
normalize : {True, False}, optional
Whether to divide by the seasonal standard deviation.
unbiased : {True, False}, optional
Whether the biased (False) or unbiased (True) estimate of the standard
deviation should be computed.
"""
if not isinstance(series, TimeSeries):
raise TypeError("The input should be a valid TimeSeries!")
season = ts.check_freq(season)
output = series.copy()
# 1.Monthly .............
if ((season // _c.FR_MTH) == 1):
input = series.asfreq(season).months
nseason = 12
# 2. Quarterly ..........
elif ((season // _c.FR_QTR) == 1):
input = series.asfreq(season).quarters
nseason = 4
# 3. Weekly .............
elif ((season // _c.FR_WK) == 1):
input = series.asfreq(season).weeks
nseason = 53
#........................
else:
raise NotImplementedError("Frequency currently supported: MTH, QTR")
#..................................
ddof = int(unbiased)
for i in range(1, nseason + 1):
selection = (input == i)
selected = series[selection]
output[selection] = current = selected.anom()
if normalize:
current /= selected.std(ddof=ddof)
return output
def deseasonalize(series, season, normalize=True, unbiased=True):
"""
Deseasonalize a time series by substracting the seasonal average and
dividing by the seasonal standard deviation (if needed).
Returns a new time series, with the same frequency as the original.
Parameters
----------
series : TimeSeries
Input time series.
season : string/integer
Frequency of the season.
For example, use 'M' or 3000 for 12 seasons per year (one per month).
normalize : {True, False}, optional
Whether to divide by the seasonal standard deviation.
unbiased : {True, False}, optional
Whether the biased (False) or unbiased (True) estimate of the standard
deviation should be computed.
"""
if not isinstance(series, TimeSeries):
raise TypeError("The input should be a valid TimeSeries!")
season = ts.check_freq(season)
output = series.copy()
# 1.Monthly .............
if ((season // _c.FR_MTH) == 1):
input = series.asfreq(season).months
nseason = 12
# 2. Quarterly ..........
elif ((season // _c.FR_QTR) == 1):
input = series.asfreq(season).quarters
nseason = 4
# 3. Weekly .............
elif ((season // _c.FR_WK) == 1):
input = series.asfreq(season).weeks
nseason = 53
#........................
else:
raise NotImplementedError("Frequency currently supported: MTH, QTR")
#..................................
ddof = int(unbiased)
for i in range(1, nseason + 1):
selection = (input == i)
selected = series[selection]
output[selection] = current = selected.anom()
if normalize:
current /= selected.std(ddof=ddof)
return output
def periodize(series, freq=None, func=None, *args, **kwargs):
"""
Convert a series to an annual frequency.
The initial series must have a monthly or quarterly frequency.
If this is not the case, a temporary series is created from the conversion
of the initial series to the ``freq`` parameter, using ``func`` as
conversion function.
Parameters
----------
series : TimeSeries
Series to convert.
freq : var, optional
A valid frequency specifier (as a string or integer), or a 3-letter
string corresponding to the first quarter of the year.
Use this parameter if the series does not have an adequate frequency.
func : {None,function}, optional
Function controlling how data sharing the same new dates should be
manipulated.
This function should handle masked values appropriately.
args : {extra arguments for func parameter}, optional
Additional mandatory parameters of the ``func`` function.
kwargs : {extra keyword arguments for func parameter}, optional
Additional optional keyword parameters of the ``func`` function.
Returns
-------
periodized_series : TimeSeries
Annual T:class:`~scikits.timeseries.TimeSeries`, with an additional key
``period_names`` key in the :attr:`optinfo` dictionary.
The name of the periods depend
on the frequency of the initial series or on the ``freq`` parameter.
Examples
--------
To group a daily series by months, with accumulation :
>>> periodize(series,"M", func=ma.sum)
To group a daily series by quarters with accumulation, the first quarter
starting in October:
>>> periodize(series, 'OND', func=ma.sum)
>>> periodize(series, 'Q-E-SEP', func=ma.sum)
To group a daily series by quarters with accumulation, the first quarter
starting in December:
>>> periodize(series, 'NDJ', func=ma.sum)
>>> periodize(series, 'Q-E-NOV', func=ma.sum)
.. todo::
This function is vestigial and has never been properly cleaned up.
Caution is required...
"""
invalidmsg = "The periodize function requires the input series "\
"to have a monthly or quarterly frequency. (got '%s' instead)"
period_convert = {
_c.FR_MTH: _c.FR_ANN,
_c.FR_QTR: _c.FR_ANN,
_c.FR_QTREDEC: _c.FR_ANNDEC,
_c.FR_QTRSJAN: _c.FR_ANNJAN,
_c.FR_QTREJAN: _c.FR_ANNJAN,
_c.FR_QTRSFEB: _c.FR_ANNFEB,
_c.FR_QTREFEB: _c.FR_ANNFEB,
_c.FR_QTRSMAR: _c.FR_ANNMAR,
_c.FR_QTREMAR: _c.FR_ANNMAR,
_c.FR_QTRSAPR: _c.FR_ANNAPR,
_c.FR_QTREAPR: _c.FR_ANNAPR,
_c.FR_QTRSMAY: _c.FR_ANNMAY,
_c.FR_QTREMAY: _c.FR_ANNMAY,
_c.FR_QTRSJUN: _c.FR_ANNJUN,
_c.FR_QTREJUN: _c.FR_ANNJUN,
_c.FR_QTRSJUL: _c.FR_ANNJUL,
_c.FR_QTREJUL: _c.FR_ANNJUL,
_c.FR_QTRSAUG: _c.FR_ANNAUG,
_c.FR_QTREAUG: _c.FR_ANNAUG,
_c.FR_QTRSSEP: _c.FR_ANNSEP,
_c.FR_QTRESEP: _c.FR_ANNSEP,
_c.FR_QTRSOCT: _c.FR_ANNOCT,
_c.FR_QTREOCT: _c.FR_ANNOCT,
_c.FR_QTRSNOV: _c.FR_ANNNOV,
_c.FR_QTRENOV: _c.FR_ANNNOV,
_c.FR_QTRSDEC: _c.FR_ANNDEC,
}
periodnames = {
_c.FR_MTH: [
'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep',
'Oct', 'Nov', 'Dec'
],
_c.FR_QTR: ['JFM', 'AMJ', 'JAS', 'OND'],
_c.FR_QTREJAN: ['FMA', 'MJJ', 'ASO', 'NDJ'],
_c.FR_QTRSJAN: ['FMA', 'MJJ', 'ASO', 'NDJ'],
_c.FR_QTREFEB: ['MAM', 'JJA', 'SON', 'DJF'],
_c.FR_QTRSFEB: ['MAM', 'JJA', 'SON', 'DJF'],
_c.FR_QTREMAR: ['AMJ', 'JAS', 'OND', 'JFM'],
_c.FR_QTRSMAR: ['AMJ', 'JAS', 'OND', 'JFM'],
_c.FR_QTREAPR: ['MJJ', 'ASO', 'NDJ', 'FMA'],
_c.FR_QTRSAPR: ['MJJ', 'ASO', 'NDJ', 'FMA'],
_c.FR_QTREMAY: ['JJA', 'SON', 'DJF', 'MAM'],
_c.FR_QTRSMAY: ['JJA', 'SON', 'DJF', 'MAM'],
_c.FR_QTREJUN: ['JAS', 'OND', 'JFM', 'AMJ'],
_c.FR_QTRSJUN: ['JAS', 'OND', 'JFM', 'AMJ'],
_c.FR_QTREJUL: ['ASO', 'NDJ', 'FMA', 'MJJ'],
_c.FR_QTRSJUL: ['ASO', 'NDJ', 'FMA', 'MJJ'],
_c.FR_QTREAUG: ['SON', 'DJF', 'MAM', 'JJA'],
_c.FR_QTRSAUG: ['SON', 'DJF', 'MAM', 'JJA'],
_c.FR_QTRESEP: ['OND', 'JFM', 'AMJ', 'JAS'],
_c.FR_QTRSSEP: ['OND', 'JFM', 'AMJ', 'JAS'],
_c.FR_QTREOCT: ['NDJ', 'FMA', 'MJJ', 'ASO'],
_c.FR_QTRSOCT: ['NDJ', 'FMA', 'MJJ', 'ASO'],
_c.FR_QTRENOV: ['DJF', 'MAM', 'JJA', 'SON'],
_c.FR_QTRSNOV: ['DJF', 'MAM', 'JJA', 'SON'],
_c.FR_QTREDEC: ['JFM', 'AMJ', 'JAS', 'OND'],
_c.FR_QTRSDEC: ['JFM', 'AMJ', 'JAS', 'OND'],
}
#
if not isinstance(series, TimeSeries):
raise TypeError("The input should be a valid TimeSeries object!")
#
if freq is None:
freq = series._dates.freq
if not (freq in period_convert):
raise ValueError(invalidmsg % series._dates.freqstr)
else:
if isinstance(freq, basestring):
freq = freq.upper()
try:
freq = check_freq(freq)
except ValueError:
months = 'JFMAMJJASONDJF'
try:
idx = months.index(freq)
except ValueError:
raise ValueError(invalidmsg % series._dates.freqstr)
freq = freqlist[idx]
else:
freq = check_freq(freq)
if not (freq in period_convert):
raise ValueError(invalidmsg % series._dates.freqstr)
if func is None:
raise ValueError("When converting to an intermediary frequency, "
"the conversion function cannot be None!")
series = series.convert(freq, func=func, *args, **kwargs)
converted = series.convert(period_convert[freq])
#
names = periodnames[freq]
output = converted
output.optinfo['period_names'] = names
# ndtype=[(_,converted.dtype.char) for _ in names]
# mdtype=[(_,bool) for _ in names]
# output = converted.__class__(np.fromiter(converted._data, dtype=ndtype),
# dates=converted._dates,
# mask=np.fromiter(ma.getmaskarray(converted),
# dtype=mdtype))
# output._update_from(converted)
return output
def periodize(series, freq=None, func=None, *args, **kwargs):
"""
Convert a series to an annual frequency.
The initial series must have a monthly or quarterly frequency.
If this is not the case, a temporary series is created from the conversion
of the initial series to the ``freq`` parameter, using ``func`` as
conversion function.
Parameters
----------
series : TimeSeries
Series to convert.
freq : var, optional
A valid frequency specifier (as a string or integer), or a 3-letter
string corresponding to the first quarter of the year.
Use this parameter if the series does not have an adequate frequency.
func : {None,function}, optional
Function controlling how data sharing the same new dates should be
manipulated.
This function should handle masked values appropriately.
args : {extra arguments for func parameter}, optional
Additional mandatory parameters of the ``func`` function.
kwargs : {extra keyword arguments for func parameter}, optional
Additional optional keyword parameters of the ``func`` function.
Returns
-------
periodized_series : TimeSeries
Annual T:class:`~scikits.timeseries.TimeSeries`, with an additional key
``period_names`` key in the :attr:`optinfo` dictionary.
The name of the periods depend
on the frequency of the initial series or on the ``freq`` parameter.
Examples
--------
To group a daily series by months, with accumulation :
>>> periodize(series,"M", func=ma.sum)
To group a daily series by quarters with accumulation, the first quarter
starting in October:
>>> periodize(series, 'OND', func=ma.sum)
>>> periodize(series, 'Q-E-SEP', func=ma.sum)
To group a daily series by quarters with accumulation, the first quarter
starting in December:
>>> periodize(series, 'NDJ', func=ma.sum)
>>> periodize(series, 'Q-E-NOV', func=ma.sum)
.. todo::
This function is vestigial and has never been properly cleaned up.
Caution is required...
"""
invalidmsg = "The periodize function requires the input series "\
"to have a monthly or quarterly frequency. (got '%s' instead)"
period_convert = {_c.FR_MTH: _c.FR_ANN, _c.FR_QTR: _c.FR_ANN,
_c.FR_QTREDEC: _c.FR_ANNDEC, _c.FR_QTRSJAN: _c.FR_ANNJAN,
_c.FR_QTREJAN: _c.FR_ANNJAN, _c.FR_QTRSFEB: _c.FR_ANNFEB,
_c.FR_QTREFEB: _c.FR_ANNFEB, _c.FR_QTRSMAR: _c.FR_ANNMAR,
_c.FR_QTREMAR: _c.FR_ANNMAR, _c.FR_QTRSAPR: _c.FR_ANNAPR,
_c.FR_QTREAPR: _c.FR_ANNAPR, _c.FR_QTRSMAY: _c.FR_ANNMAY,
_c.FR_QTREMAY: _c.FR_ANNMAY, _c.FR_QTRSJUN: _c.FR_ANNJUN,
_c.FR_QTREJUN: _c.FR_ANNJUN, _c.FR_QTRSJUL: _c.FR_ANNJUL,
_c.FR_QTREJUL: _c.FR_ANNJUL, _c.FR_QTRSAUG: _c.FR_ANNAUG,
_c.FR_QTREAUG: _c.FR_ANNAUG, _c.FR_QTRSSEP: _c.FR_ANNSEP,
_c.FR_QTRESEP: _c.FR_ANNSEP, _c.FR_QTRSOCT: _c.FR_ANNOCT,
_c.FR_QTREOCT: _c.FR_ANNOCT, _c.FR_QTRSNOV: _c.FR_ANNNOV,
_c.FR_QTRENOV: _c.FR_ANNNOV, _c.FR_QTRSDEC: _c.FR_ANNDEC,
}
periodnames = {_c.FR_MTH:['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun',
'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec'],
_c.FR_QTR:['JFM', 'AMJ', 'JAS', 'OND'],
_c.FR_QTREJAN:['FMA', 'MJJ', 'ASO', 'NDJ'], _c.FR_QTRSJAN:['FMA', 'MJJ', 'ASO', 'NDJ'],
_c.FR_QTREFEB:['MAM', 'JJA', 'SON', 'DJF'], _c.FR_QTRSFEB:['MAM', 'JJA', 'SON', 'DJF'],
_c.FR_QTREMAR:['AMJ', 'JAS', 'OND', 'JFM'], _c.FR_QTRSMAR:['AMJ', 'JAS', 'OND', 'JFM'],
_c.FR_QTREAPR:['MJJ', 'ASO', 'NDJ', 'FMA'], _c.FR_QTRSAPR:['MJJ', 'ASO', 'NDJ', 'FMA'],
_c.FR_QTREMAY:['JJA', 'SON', 'DJF', 'MAM'], _c.FR_QTRSMAY:['JJA', 'SON', 'DJF', 'MAM'],
_c.FR_QTREJUN:['JAS', 'OND', 'JFM', 'AMJ'], _c.FR_QTRSJUN:['JAS', 'OND', 'JFM', 'AMJ'],
_c.FR_QTREJUL:['ASO', 'NDJ', 'FMA', 'MJJ'], _c.FR_QTRSJUL:['ASO', 'NDJ', 'FMA', 'MJJ'],
_c.FR_QTREAUG:['SON', 'DJF', 'MAM', 'JJA'], _c.FR_QTRSAUG:['SON', 'DJF', 'MAM', 'JJA'],
_c.FR_QTRESEP:['OND', 'JFM', 'AMJ', 'JAS'], _c.FR_QTRSSEP:['OND', 'JFM', 'AMJ', 'JAS'],
_c.FR_QTREOCT:['NDJ', 'FMA', 'MJJ', 'ASO'], _c.FR_QTRSOCT:['NDJ', 'FMA', 'MJJ', 'ASO'],
_c.FR_QTRENOV:['DJF', 'MAM', 'JJA', 'SON'], _c.FR_QTRSNOV:['DJF', 'MAM', 'JJA', 'SON'],
_c.FR_QTREDEC:['JFM', 'AMJ', 'JAS', 'OND'], _c.FR_QTRSDEC:['JFM', 'AMJ', 'JAS', 'OND'],
}
#
if not isinstance(series, TimeSeries):
raise TypeError("The input should be a valid TimeSeries object!")
#
if freq is None:
freq = series._dates.freq
if not (freq in period_convert):
raise ValueError(invalidmsg % series._dates.freqstr)
else:
if isinstance(freq, basestring):
freq = freq.upper()
try:
freq = check_freq(freq)
except ValueError:
months = 'JFMAMJJASONDJF'
try:
idx = months.index(freq)
except ValueError:
raise ValueError(invalidmsg % series._dates.freqstr)
freq = freqlist[idx]
else:
freq = check_freq(freq)
if not (freq in period_convert):
raise ValueError(invalidmsg % series._dates.freqstr)
if func is None:
raise ValueError("When converting to an intermediary frequency, "
"the conversion function cannot be None!")
series = series.convert(freq, func=func, *args, **kwargs)
converted = series.convert(period_convert[freq])
#
names = periodnames[freq]
output = converted
output.optinfo['period_names'] = names
# ndtype=[(_,converted.dtype.char) for _ in names]
# mdtype=[(_,bool) for _ in names]
# output = converted.__class__(np.fromiter(converted._data, dtype=ndtype),
# dates=converted._dates,
# mask=np.fromiter(ma.getmaskarray(converted),
# dtype=mdtype))
# output._update_from(converted)
return output
