def _multiple_window(self, stime, etime, step):
""" Create iterator of multiple time window for
data set more than DSS_MAX_RTS_POINTS.
"""
lt = []
stimet = stime
while stimet < etime:
nval = DSS_MAX_RTS_POINTS
try:
etimet = increment(stimet, step, DSS_MAX_RTS_POINTS)
except: ## in case when step is big like month, DSS_MAX_RTS_POINTS will
## cause over reasonable year range. Such case should use
## one windows
etimet = etime
nval = number_intervals(stimet, etimet, step)
if etimet > etime:
etimet = etime
nval = number_intervals(stimet, etimet, step)
cdate = stimet.date()
cdate = cdate.strftime('%d%b%Y')
ctime = stimet.time()
ctime = ctime.strftime('%H%M')
stimet = etimet
lt.append((cdate, ctime, nval))
return iter(lt)
def test_merge_rts_no_intersect(self):
""" Test merging two rts without intersection."""
d1=[1]*int(spy.math.pow(2,10))
d2=[2]*int(spy.math.pow(2,11))
st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
st2=datetime.datetime(year=1990,month=5,day=3,hour=11, minute=15)
ts1=rts(d1,st1,time_interval(hours=1))
ts2=rts(d2,st2,time_interval(hours=1))
nt=merge(ts1,ts2)
self.assertEqual(nt.start,ts1.start)
self.assertEqual(nt.end,ts2.end)
total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
self.assertEqual(len(nt.data),total_n)
s1=nt.index_after(ts1.end)
s2=nt.index_after(ts2.start)
self.assert_(spy.alltrue(spy.isnan(nt.data[s1+1:s2])))
self.assert_(spy.allclose(nt.data[0:s1+1],ts1.data))
self.assert_(spy.allclose(nt.data[s2:len(nt.data)],ts2.data))
def test_merge_rts_intersect(self):
""" Test merging two rts with intersection."""
d1=[1.0]*int(spy.math.pow(2,10))
d2=[2.0]*int(spy.math.pow(2,11))
st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
st2=datetime.datetime(year=1990,month=3,day=3,hour=11, minute=15)
ts1=rts(d1,st1,time_interval(hours=1))
ts2=rts(d2,st2,time_interval(hours=1))
## Intentionally put some nan into ts1 to see effect.
ii=ts1.index_after(st2)
num_nan=10
ts1.data[ii+1:ii+num_nan+2]=spy.nan
nt=merge(ts1,ts2)
self.assertEqual(nt.start,ts1.start)
self.assertEqual(nt.end,ts2.end)
total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
self.assertEqual(len(nt.data),total_n)
s1=nt.index_after(ts1.end)
s2=nt.index_after(ts2.start)
s3=ts2.index_after(ts1.end)
self.assert_(spy.allclose(nt.data[0:ii+1],ts1.data[0:ii+1]))
self.assert_(spy.allclose(nt.data[ii+1:ii+num_nan+2],ts2.data[0:num_nan+1]))
self.assert_(spy.allclose(nt.data[ii+num_nan+2:s1+1],ts1.data[ii+num_nan+2:s1+1]))
self.assert_(spy.allclose(nt.data[s1+1:len(nt.data)],ts2.data[s3+1:len(ts2.data)]))
## a small test
d1=[1.0]*4
d2=[2.0]*2
st1=datetime.datetime(year=1990,month=2,day=1)
st2=datetime.datetime(year=1990,month=3,day=1)
ts1=rts(d1,st1,time_interval(months=1))
ts2=rts(d2,st2,time_interval(months=1))
nt=merge(ts1,ts2)
self.assertEqual(len(nt),len(ts1))
self.assertEqual(nt.data[-1],ts1.data[-1])
d1=[1.0,1.0,1.0,spy.nan]
d2=[2.0,3.0,4.0]
st1=datetime.datetime(year=1990,month=2,day=1)
st2=datetime.datetime(year=1990,month=3,day=1)
ts1=rts(d1,st1,time_interval(months=1))
ts2=rts(d2,st2,time_interval(months=1))
nt=merge(ts1,ts2)
self.assertEqual(len(nt),len(ts1))
self.assertEqual(nt.data[-1],ts2.data[-1])
def test_merge_rts_2d_intersect(self):
""" Test merging of two rts of 2-d data and has intersection."""
d1=[[1.0,2.0]]*int(spy.math.pow(2,10))
d2=[[3.0,4.0]]*int(spy.math.pow(2,11))
st1=datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15)
st2=datetime.datetime(year=1990,month=3,day=3,hour=11, minute=15)
ts1=rts(d1,st1,time_interval(hours=1))
ts2=rts(d2,st2,time_interval(hours=1))
## Intentionally put some nan into ts1 to see effect.
ii=ts1.index_after(st2)
num_nan=10
ts1.data[ii+1:ii+num_nan+2,]=spy.nan
nt=merge(ts1,ts2)
self.assertEqual(nt.start,ts1.start)
self.assertEqual(nt.end,ts2.end)
total_n=number_intervals(ts1.start,ts2.end,ts1.interval)+1
self.assertEqual(len(nt.data),total_n)
s1=nt.index_after(ts1.end)
s2=nt.index_after(ts2.start)
s3=ts2.index_after(ts1.end)
self.assert_(spy.allclose(nt.data[0:ii+1,],ts1.data[0:ii+1,]))
self.assert_(spy.allclose(nt.data[ii+1:ii+num_nan+2,],ts2.data[0:num_nan+1,]))
self.assert_(spy.allclose(nt.data[ii+num_nan+2:s1+1,],ts1.data[ii+num_nan+2:s1+1,]))
self.assert_(spy.allclose(nt.data[s1+1:len(nt.data),],ts2.data[s3+1:len(ts2.data),]))
def _bind(ts1, ts2):
""" bind data from timeseries ts1 and ts2.
Parameters
----------
ts1,ts2 : :class:`~vtools.data.timeseries.TimeSeries`
Two timeseries
Returns
-------
merged : :class:`~vtools.data.timeseries.TimeSeries`
A new binded time series if success.
"""
if (not ((ts1.data.ndim == 1) and (ts2.data.ndim == 1))):
raise ValueError("bind only support time series of univariate")
ts = None
ts_is_regular = False
new_ts_time_sequence = []
new_start = None
new_interval = None
if ((ts1.is_regular()) and (ts2.is_regular())):
ts1_start = ts1.times[0]
ts1_end = ts1.times[-1]
ts2_start = ts2.times[0]
ts2_end = ts2.times[-1]
new_start = ts1_start
if new_start > ts2_start:
new_start = ts2_start
new_end = ts1_end
if new_end < ts2_end:
new_end = ts2_end
new_interval = ts1.interval
ts2_interval = ts2.interval
if new_interval > ts2_interval:
new_interval = ts2_interval
num_data = number_intervals(new_start, new_end, new_interval) + 1
new_ts_time_sequence = time_sequence(new_start, new_interval, num_data)
ts_is_regular = True
else:
new_ts_time_sequence = np.union1d(ts1.ticks, ts2.ticks)
new_ts_len = len(new_ts_time_sequence)
new_data = np.array([[np.nan] * new_ts_len, [np.nan] * new_ts_len])
ts1_data_id = np.searchsorted(new_ts_time_sequence, ts1.ticks)
ts2_data_id = np.searchsorted(new_ts_time_sequence, ts2.ticks)
new_data[0, ts1_data_id] = ts1.data
new_data[1, ts2_data_id] = ts2.data
new_data = new_data.transpose()
if ts_is_regular:
ts = rts(new_data, new_start, new_interval)
else:
ts = its(new_ts_time_sequence, new_data)
return ts
def test_decimate_rts(self):
# Test operations on ts of varied values.
test_ts=[(datetime.datetime(year=1990,month=2,day=3,hour=11, minute=15),
int(scipy.math.pow(2,10)),time_interval(minutes=5)),]
for (st,num,delta) in test_ts:
data=[scipy.math.sin(0.01*scipy.math.pi*k) for k in range(num)]
# This ts is the orignial one
ts0=rts(data,st,delta,{})
for interval in self.test_interval:
sample_num=number_intervals(ts0.start,ts0.end,interval)+1
ts=decimate(ts0,interval)
self.assert_(ts.is_regular())
self.assert_(len(ts.data)==sample_num)
def _interpolate2rts(ts, interval, method=SPLINE, filter_nan=True, **dic):
""" Interpolate a time series to get a regular series with input `interval`.
Parameters
-----------
ts : :class:`~vtools.data.timeseries.TimeSeries`
Series to be interpolated
method : string, optional
See :func:`interpolate_ts_nan` for a list of methods
filter_nan : boolean,optional
True if nan points should be omitted or not.
**dic : Dictionary
Dictonary of extra parameters to be passed to `method`
Returns
-------
result: :class:`~vtools.data.timeseries.TimeSeries`
A regular time series.
"""
if type(interval) == type(" "):
interval = parse_interval(interval)
if not is_interval(interval):
raise TypeError("second input must be a time interval or can be "
"parsed as time interval")
#if ts.is_regular():
# if ts.start+interval>ts.start+ts.interval:
# raise ValueError("interpolating interval is wider than"
# "input time series' interval")
rt_start = align(ts.start, interval, 1)
rt_end = align(ts.end, interval, -1)
num = number_intervals(rt_start, rt_end, interval) + 1
times = time_sequence(rt_start, interval, num)
values = _interpolate_ts2array(ts, times, method, filter_nan, **dic)
prop = deepcopy(ts.props)
prop[AGGREGATION] = INDIVIDUAL
prop[TIMESTAMP] = INST
rt = rts(values, rt_start, interval, prop)
return rt
def test_period_op(self):
## Test operations on ts of varied values.
test_input = [
(datetime.datetime(year=1990, month=2, day=3,
hour=11, minute=15), 3005,
time_interval(minutes=5), "1hour", time_interval(hours=1),
datetime.datetime(year=1990, month=2, day=3, hour=12, minute=0)),
(datetime.datetime(year=1990, month=2,
day=3, hour=11, minute=15), 3301,
time_interval(days=1), "1month", time_interval(months=1),
datetime.datetime(year=1990, month=3, day=1, hour=00, minute=00)),
(datetime.datetime(year=1990, month=2,
day=3, hour=11, minute=15), 3407,
time_interval(hours=1), "1day", time_interval(days=1),
datetime.datetime(year=1990, month=2, day=4, hour=00, minute=00)),
(datetime.datetime(year=1990, month=2,
day=3, hour=11, minute=45), 6093,
time_interval(days=1), "1year", time_interval(years=1),
datetime.datetime(year=1991, month=1, day=1, hour=00, minute=00)),
(datetime.datetime(year=1990, month=1,
day=1, hour=00, minute=00), 10957,
time_interval(days=1), "1year", time_interval(years=1),
datetime.datetime(year=1990, month=1, day=1, hour=00, minute=00)),
(datetime.datetime(year=1990, month=1,
day=1, hour=00, minute=00), 10957,
time_interval(days=1), "7 day", time_interval(days=7),
datetime.datetime(year=1990, month=1, day=1, hour=00, minute=00)),
]
for (st, num, delta, interval, op_delta, aligned_start) in test_input:
data=[random.uniform(self.min_val,self.max_val) \
for k in range(num)]
ts = rts(data, st, delta, {})
## The length of new generated time series.
nt_len = number_intervals(aligned_start, ts.end, op_delta)
for op in [MIN, MAX, SUM]:
nt = period_op(ts, interval, op)
self.assertEqual(nt.start, aligned_start)
self.assertEqual(len(nt.data), nt_len)
for op in [MEAN]:
for method in [RECT, TRAPEZOID]:
nt = period_op(ts, interval, op, method=method)
self.assertEqual(nt.start, aligned_start)
self.assertEqual(nt.props[TIMESTAMP], PERIOD_START)
self.assertEqual(nt.props[AGGREGATION], op)
self.assertEqual(len(nt.data), nt_len)
## Test operations on ts of constant values.
data = [1 for k in range(num)]
ts = rts(data, st, delta, {})
## This is the data values that new ts should be
nt_data = [1] * nt_len
#pdb.set_trace()
for op in [MEAN]:
for method in [RECT, TRAPEZOID]:
nt = period_op(ts, interval, op, method=method)
l = min(len(nt.data), len(nt_data))
assert_array_equal(nt.data[0:l],nt_data[0:l],\
err_msg="two array not equal in averaging" \
" by %s"%(method))
def resample(ts, interval, aligned=True):
""" Do a resampling operation on a time series.
Parameters
----------
ts : :class:`~vtools.data.timeseries.TimeSeries`
A regular timeseries to be resampled.
interval : :ref:`time_interval <time_intervals>`
Interval of resampled time series.
aligned: boolean
Default is true which means aligning result' time with input 'interval'.
Returns
-------
Resampled : :class:`~vtools.data.timeseries.TimeSeries`
A new resampled time series if success.
Raises
--------
error : :py:class:`ValueError`
If input time series is not regular, or regular interval is calendar
dependent.
.. note::This is a simple resample method with side effect of
aliasing.
"""
# Resampling regular time series with calendar dependent
# interval is not allowed for the time being.
if ts.is_regular():
tsdelta = ts.interval
if is_calendar_dependent(tsdelta):
raise ValueError("Resampling of a regular time series \
with calendar dependent interval is not defined.")
else:
raise ValueError("input timeseris for resampling operation \
must be regualr timeserise with calendar independent interval.")
if not is_interval(interval):
interval = parse_interval(interval)
if aligned:
aligned_start = align(ts.start, interval, 1)
return resample(ts.window(aligned_start, ts.end), interval, False)
else:
num = number_intervals(ts.start, ts.end, interval)
steps = int(ticks(interval) / ticks(ts.interval))
nt = ts.data[0:num * steps + 1:steps, ]
st = ts.start
prop = {}
for key, val in ts.props.items():
prop[key] = val
prop[TIMESTAMP] = INST
prop[AGGREGATION] = INDIVIDUAL
new_ts = rts(nt, st, interval, prop)
return new_ts
def decimate(ts, interval, **dic):
""" Downsample timeseries by decimating.
A wrapper of GNU Octave function decimate.m function.
This will remove the influence of aliasing on downsampled
ts.
Parameters
----------
ts : :class:`~vtools.data.timeseries.TimeSeries`
A regular timeseries to be resampled.
interval : :ref:`time_interval <time_intervals>`
Interval of resampled time series.
**dic : Dictionary
Dictonary of extra parameters to be passed in. For instance , input
'align_calendar' as boolean to align resampled timeseries with the
downsampling interval.
Returns
-------
Resampled : :class:`~vtools.data.timeseries.TimeSeries`
A new resampled time series if success.
"""
if "align_calendar" in dic.keys():
align_calendar = dic["align_calendar"]
del dic["align_calendar"]
else:
align_calendar = False
if not is_interval(interval):
interval = parse_interval(interval)
if align_calendar:
resample_start = align(ts.start, interval, 1)
else:
resample_start = ts.start
new_num = number_intervals(resample_start, ts.end, interval)
len_ts = number_intervals(resample_start, ts.end, ts.interval)
n = int(len_ts / new_num)
if n < 1:
raise ValueError("Input downsampling interval \
is narrower than orginal timeseries,try a wider one.")
if ts.data.ndim == 1:
data, shift = _decimate(ts.data, n, **dic)
# d1=data[::-1].copy()
# d2=_decimate(d1,1,**dic)
# data=d2[::-1].copy()
elif ts.data.ndim == 2:
data, shift = _decimate(ts.data, n, axis=0, **dic)
# d1=data[::-1].copy()
# d2=_decimate(d1,1,axis=0,**dic)
# data=d2[::-1].copy()
#shift is in time inteval
interval_ticks = ticks(ts.interval)
left_shift = interval_ticks * shift
resample_start = resample_start - ticks_to_interval(left_shift)
ll = data.shape[0]
prop = {}
for key, val in ts.props.items():
prop[key] = val
prop[TIMESTAMP] = INST
prop[AGGREGATION] = INDIVIDUAL
new_rts = rts(data, resample_start, interval, prop)
return new_rts