def test_ewma_span_com_args(self):
A = moments.ewma(self.arr, com=9.5)
B = moments.ewma(self.arr, span=20)
assert_almost_equal(A, B)
self.assertRaises(Exception, moments.ewma, self.arr, com=9.5, span=20)
self.assertRaises(Exception, moments.ewma, self.arr)
def test_ewma_span_com_args(self):
A = mom.ewma(self.arr, com=9.5)
B = mom.ewma(self.arr, span=20)
assert_almost_equal(A, B)
self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, span=20)
self.assertRaises(Exception, mom.ewma, self.arr)
def test_ewma_halflife_arg(self):
A = mom.ewma(self.arr, com=13.932726172912965)
B = mom.ewma(self.arr, halflife=10.0)
assert_almost_equal(A, B)
self.assertRaises(Exception, mom.ewma, self.arr, span=20, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, span=20, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr)
def test_ewma_halflife_arg(self):
A = mom.ewma(self.arr, com=13.932726172912965)
B = mom.ewma(self.arr, halflife=10.0)
assert_almost_equal(A, B)
self.assertRaises(Exception, mom.ewma, self.arr, span=20, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, span=20, halflife=50)
self.assertRaises(Exception, mom.ewma, self.arr)
def compute_macd(data):
data[’ewma5’] = ewma(data.close, span=5)
data[’ewma35’] = ewma(data.close, span=35)
data[’macd’] = data.ewma5 - data.ewma35
data[’macd_signal’] = ewma(data.macd, span=5)
data[’histogram’] = data.macd - data.macd_signal
def moving_average_convergence(x, nslow=50, nfast=10):
"""
compute the crossover (Moving Average Convergence/Divergence) using a fast and slow exponential moving avg'
return value is emaslow, emafast, crossover which are len(x) arrays
"""
emaslow = mom.ewma(x, span=nslow)
emafast = mom.ewma(x, span=nfast)
# emaslow = moving_average(x, nslow, type='exponential')
# emafast = moving_average(x, nfast, type='exponential')
# print x
# time.sleep(10)
return emaslow, emafast, emafast - emaslow
def moving_average_convergence(x, nslow=50, nfast=10):
"""
compute the crossover (Moving Average Convergence/Divergence) using a fast and slow exponential moving avg'
return value is emaslow, emafast, crossover which are len(x) arrays
"""
emaslow = mom.ewma(x, span=nslow)
emafast = mom.ewma(x, span=nfast)
# emaslow = moving_average(x, nslow, type='exponential')
# emafast = moving_average(x, nfast, type='exponential')
# print x
# time.sleep(10)
return emaslow, emafast, emafast - emaslow
def calc_rtg_daily(daily_df, horizon):
print "Caculating daily rtg..."
result_df = filter_expandable(daily_df)
print "Calculating rtg0..."
halflife = horizon / 2
# result_df['dk'] = np.exp( -1.0 * halflife * (result_df['gdate'] - result_df['last']).astype('timedelta64[D]').astype(int) )
result_df['cum_ret'] = pd.rolling_sum(result_df['log_ret'], horizon)
result_df['sum'] = result_df['mean'] * result_df['count']
result_df['det_diff'] = result_df['sum'].diff()
result_df['det_diff_dk'] = ewma(result_df['det_diff'], halflife=horizon )
result_df['rtg0'] = result_df['det_diff_dk'] * result_df['det_diff_dk']
# result_df['median'] = -1.0 * (result_df['median'] - 3)
# result_df['med_diff'] = result_df['median'].unstack().diff().stack()
# result_df['med_diff_dk'] = pd.rolling_sum( result_df['dk'] * result_df['med_diff'], window=horizon )
# result_df['rtg0'] = (np.sign(result_df['med_diff_dk']) * np.sign(result_df['cum_ret'])).clip(lower=0) * result_df['med_diff_dk']
demean = lambda x: (x - x.mean())
indgroups = result_df[['rtg0', 'gdate', 'ind1']].groupby(['gdate', 'ind1'], sort=True).transform(demean)
result_df['rtg0_ma'] = indgroups['rtg0']
# result_df['rtg0_ma'] = result_df['rtg0_ma'] * (np.sign(result_df['rtg0_ma']) * np.sign(result_df['cum_ret']))
# result_df['rtg0_ma'] = result_df['rtg0']
shift_df = result_df.unstack().shift(1).stack()
result_df['rtg1_ma'] = shift_df['rtg0_ma']
return result_df
def exp_smoothing(df):
'''
Function to calculate exponential smoothing
Pandas functions are used to calculate moving average and standard deviation
Upper and lower thresholds are calculated by
Upper threshold = moving average + standard deviation
Lower threshold = moving average - standard deviation
'''
#Calculate exp weighted moving average
ExpWeightedMovingAvg = ewma(df,span=N)
ExpWeightedMovingAvg.columns=['Exp Weighted Moving Avg']
#Calculate exp weighted moving std
ExpWeightedMovingStd = ewmstd(df,span=N)
ExpWeightedMovingStd.columns=['Std Deviation']
s1=df['Original data']
s2=ExpWeightedMovingAvg['Exp Weighted Moving Avg']
s3=ExpWeightedMovingStd['Std Deviation']
s4=s2.add(s3, fill_value=0)
s5=s2.sub(s3, fill_value=0)
df2=DataFrame(s1,columns=['Original data'])
df2['Exp Weighted Moving Avg']=s2
df2['Std Deviation']=s3
df2['Upper Threshold']=s4
df2['Lower Threshold']=s5
return df2
def exp_smoothing(df):
'''
Function to calculate exponential smoothing
Pandas functions are used to calculate moving average and standard deviation
Upper and lower thresholds are calculated by
Upper threshold = moving average + standard deviation
Lower threshold = moving average - standard deviation
'''
#Calculate exp weighted moving average
ExpWeightedMovingAvg = ewma(df, span=N)
ExpWeightedMovingAvg.columns = ['Exp Weighted Moving Avg']
#Calculate exp weighted moving std
ExpWeightedMovingStd = ewmstd(df, span=N)
ExpWeightedMovingStd.columns = ['Std Deviation']
s1 = df['Original data']
s2 = ExpWeightedMovingAvg['Exp Weighted Moving Avg']
s3 = ExpWeightedMovingStd['Std Deviation']
s4 = s2.add(s3, fill_value=0)
s5 = s2.sub(s3, fill_value=0)
df2 = DataFrame(s1, columns=['Original data'])
df2['Exp Weighted Moving Avg'] = s2
df2['Std Deviation'] = s3
df2['Upper Threshold'] = s4
df2['Lower Threshold'] = s5
return df2
def pd_ema(arg, ratio=0.1):
''' EMA, implemented with `pandas.stats.moments.ewma` '''
span = 2.0 / (1-ratio) - 1
arg = utils.safe_series(arg)
rval = ewma(arg, span=span)
utils.safe_name(rval, name='pdEMA')
return rval
def test_ewma(self):
self._check_ew(mom.ewma)
arr = np.zeros(1000)
arr[5] = 1
result = mom.ewma(arr, span=100, adjust=False).sum()
self.assert_(np.abs(result - 1) < 1e-2)
def test_ewma(self):
self._check_ew(mom.ewma)
arr = np.zeros(1000)
arr[5] = 1
result = mom.ewma(arr, span=100, adjust=False).sum()
self.assert_(np.abs(result - 1) < 1e-2)
def ewma(params, df):
''' Ewmas specified columns of the df '''
print('Ewma data')
#unknownFeatures = features['unknowns']
#knownFeatures = features['knowns']
dfX, dfy = df2xy(df, params.data_cols.features, params.data_cols.target_col)
resultDf = pd.DataFrame()
for col in dfX.columns:
series = dfX[col]
# take EWMA in both directions with a smaller span term
fwd = moments.ewma(series, span=params.hypers.days_to_average) # take EWMA in fwd direction
#bwd = ewma( series[::-1], span=self.daysToAverage ) # take EWMA in bwd direction
#c = np.vstack(( fwd, bwd[::-1] )) # lump fwd and bwd together
#c = np.mean( c, axis=0 ) # average
c = fwd
cDf = pd.DataFrame(c,columns=[col])
newDf = pd.concat([resultDf, cDf], axis=1, ignore_index=True)
columns = list(resultDf.columns)+[col]
newDf.columns = columns
resultDf = newDf
# Attach ewma'd Xs with y and send back df
dfX = resultDf.round(1)
df = xy2df(dfX, dfy)
return df
def mpl_plot_ewma_embedded(figname, xs, ys, span):
# create the figure and axes for the plot
fig = Figure(figsize=(8, 6), dpi=75, facecolor=(1, 1, 1), edgecolor=(0, 0, 0))
ax = fig.add_subplot(111)
# calculate the moving average
ewma_ys = ewma(ys, span=span)
# plot the data
ax.plot(xs, ys, alpha=0.4, label="Raw")
ax.plot(xs, ewma_ys, label="EWMA")
ax.legend()
# write the figure to a temporary image file
filename = os.path.join(os.environ["TEMP"], "xlplot_%s.png" % figname)
canvas = FigureCanvas(fig)
canvas.draw()
canvas.print_png(filename)
# Show the figure in Excel as a Picture object on the same sheet
# the function is being called from.
xl = xl_app()
caller = xlfCaller()
sheet = xl.Range(caller.address).Worksheet
# if a picture with the same figname already exists then get the position
# and size from the old picture and delete it.
for old_picture in sheet.Pictures():
if old_picture.Name == figname:
height = old_picture.Height
width = old_picture.Width
top = old_picture.Top
left = old_picture.Left
old_picture.Delete()
break
else:
# otherwise place the picture below the calling cell.
top_left = sheet.Cells(caller.rect.last_row+2, caller.rect.last_col+1)
top = top_left.Top
left = top_left.Left
width, height = fig.bbox.bounds[2:]
# insert the picture
# Ref: http://msdn.microsoft.com/en-us/library/office/ff198302%28v=office.15%29.aspx
picture = sheet.Shapes.AddPicture(Filename=filename,
LinkToFile=0, # msoFalse
SaveWithDocument=-1, # msoTrue
Left=left,
Top=top,
Width=width,
Height=height)
# set the name of the new picture so we can find it next time
picture.Name = figname
# delete the temporary file
os.unlink(filename)
return "[Plotted '%s']" % figname
def simpleMApredict(x,span,periods = pred_period):
x_predict = np.zeros((span+periods,))
x_predict[:span] = x[-span:]
pred = ewma(x_predict,span)[span:]
pred = pred.round()
pred[pred < 0] = 0
return pred
def predict(x, span, periods=pred_period):
x_predict = np.zeros((span + periods, ))
x_predict[:span] = x[-span:]
pred = ewma(x_predict, span)[span:]
pred = pred.round()
pred[pred < 0] = 0
return pred
def predict_EWMA(x,span=3,periods = pred_period):
x_predict = np.zeros((span+periods,))
x_predict[:span] = x[-span:]
pred = ewma(x_predict,span)[span:]
pred = pred.round()
pred[pred < 0] = 0
return pred
def McClellanOscillator():
"""
McClellan Oscillator
http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:mcclellan_oscillator
"""
def RANA():
"""
Ratio Adjusted Net Advances (RANA)
Returns RANA for whole dataset
"""
return ((data['Advances'] - data['Declines']) / (data['Advances'] + data['Declines']) * 1000)
####### Calculate RANA
rana = RANA()
####### Calculate EMAS
ema19 = ewma(rana, span=19)
ema39 = ewma(rana, span=39)
return ema19 - ema39
def mpl_plot_ewma(figname, xs, ys, span):
"""
Show a matplotlib line plot of xs vs ys and ewma(ys, span) in an interactive window.
:param figname: name to use for this plot's window
:param xs: list of x values as a column
:param ys: list of y values as a column
:param span: ewma span
"""
# Get the Qt app.
# Note: no need to 'exec' this as it will be polled in the main windows loop.
app = get_qt_app()
# create the figure and axes for the plot
fig = Figure(figsize=(600, 600),
dpi=72,
facecolor=(1, 1, 1),
edgecolor=(0, 0, 0))
ax = fig.add_subplot(111)
# calculate the moving average
ewma_ys = ewma(ys, span=span)
# plot the data
ax.plot(xs, ys, alpha=0.4, label="Raw")
ax.plot(xs, ewma_ys, label="EWMA")
ax.legend()
# generate the canvas to display the plot
canvas = FigureCanvas(fig)
# Get or create the Qt windows to show the chart in.
if figname in _plot_windows:
# get from the global dict and clear any previous widgets
window = _plot_windows[figname]
layout = window.layout()
if layout:
for i in reversed(range(layout.count())):
layout.itemAt(i).widget().setParent(None)
else:
# create a new window for this plot and store it for next time
window = QtGui.QWidget()
window.resize(800, 600)
window.setWindowTitle(figname)
_plot_windows[figname] = window
# create the navigation toolbar
toolbar = NavigationToolbar(canvas, window)
# add the canvas and toolbar to the window
layout = window.layout() or QtGui.QVBoxLayout()
layout.addWidget(canvas)
layout.addWidget(toolbar)
window.setLayout(layout)
window.show()
return "[Plotted '%s']" % figname
def test_ewma(self):
self._check_ew(mom.ewma)
arr = np.zeros(1000)
arr[5] = 1
result = mom.ewma(arr, span=100, adjust=False).sum()
self.assertTrue(np.abs(result - 1) < 1e-2)
s = Series([1.0, 2.0, 4.0, 8.0])
expected = Series([1.0, 1.6, 2.736842, 4.923077])
for f in [
lambda s: mom.ewma(s, com=2.0, adjust=True),
lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=False),
lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=True),
]:
result = f(s)
assert_series_equal(result, expected)
expected = Series([1.0, 1.333333, 2.222222, 4.148148])
for f in [
lambda s: mom.ewma(s, com=2.0, adjust=False),
lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=False),
lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=True),
]:
result = f(s)
assert_series_equal(result, expected)
def test_ewma(self):
self._check_ew(mom.ewma)
arr = np.zeros(1000)
arr[5] = 1
result = mom.ewma(arr, span=100, adjust=False).sum()
self.assertTrue(np.abs(result - 1) < 1e-2)
s = Series([1.0, 2.0, 4.0, 8.0])
expected = Series([1.0, 1.6, 2.736842, 4.923077])
for f in [lambda s: mom.ewma(s, com=2.0, adjust=True),
lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=False),
lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=True),
]:
result = f(s)
assert_series_equal(result, expected)
expected = Series([1.0, 1.333333, 2.222222, 4.148148])
for f in [lambda s: mom.ewma(s, com=2.0, adjust=False),
lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=False),
lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=True),
]:
result = f(s)
assert_series_equal(result, expected)
def test_ewma_nan_handling(self):
s = Series([1.] + [np.nan] * 5 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [1.] * len(s))
s = Series([np.nan] * 2 + [1.] + [np.nan] * 2 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [np.nan] * 2 + [1.] * 4)
# GH 7603
s0 = Series([np.nan, 1., 101.])
s1 = Series([1., np.nan, 101.])
s2 = Series([np.nan, 1., np.nan, np.nan, 101., np.nan])
com = 2.
alpha = 1. / (1. + com)
def simple_wma(s, w):
return (s.multiply(w).cumsum() / w.cumsum()).fillna(method='ffill')
for (s, adjust, ignore_na, w) in [
(s0, True, False, [np.nan, (1.0 - alpha), 1.]),
(s0, True, True, [np.nan, (1.0 - alpha), 1.]),
(s0, False, False, [np.nan, (1.0 - alpha), alpha]),
(s0, False, True, [np.nan, (1.0 - alpha), alpha]),
(s1, True, False, [(1.0 - alpha)**2, np.nan, 1.]),
(s1, True, True, [(1.0 - alpha), np.nan, 1.]),
(s1, False, False, [(1.0 - alpha)**2, np.nan, alpha]),
(s1, False, True, [(1.0 - alpha), np.nan, alpha]),
(s2, True, False,
[np.nan, (1.0 - alpha)**3, np.nan, np.nan, 1., np.nan]),
(s2, True, True,
[np.nan, (1.0 - alpha), np.nan, np.nan, 1., np.nan]),
(s2, False, False,
[np.nan, (1.0 - alpha)**3, np.nan, np.nan, alpha, np.nan]),
(s2, False, True,
[np.nan, (1.0 - alpha), np.nan, np.nan, alpha, np.nan]),
]:
expected = simple_wma(s, Series(w))
result = mom.ewma(s, com=com, adjust=adjust, ignore_na=ignore_na)
assert_series_equal(result, expected)
if ignore_na is False:
# check that ignore_na defaults to False
result = mom.ewma(s, com=com, adjust=adjust)
assert_series_equal(result, expected)
def mpl_plot_ewma(figname, xs, ys, span):
"""
Show a matplotlib line plot of xs vs ys and ewma(ys, span) in an interactive window.
:param figname: name to use for this plot's window
:param xs: list of x values as a column
:param ys: list of y values as a column
:param span: ewma span
"""
# Get the Qt app.
# Note: no need to 'exec' this as it will be polled in the main windows loop.
app = get_qt_app()
# create the figure and axes for the plot
fig = Figure(figsize=(600, 600), dpi=72, facecolor=(1, 1, 1), edgecolor=(0, 0, 0))
ax = fig.add_subplot(111)
# calculate the moving average
ewma_ys = ewma(ys, span=span)
# plot the data
ax.plot(xs, ys, alpha=0.4, label="Raw")
ax.plot(xs, ewma_ys, label="EWMA")
ax.legend()
# generate the canvas to display the plot
canvas = FigureCanvas(fig)
# Get or create the Qt windows to show the chart in.
if figname in _plot_windows:
# get from the global dict and clear any previous widgets
window = _plot_windows[figname]
layout = window.layout()
if layout:
for i in reversed(range(layout.count())):
layout.itemAt(i).widget().setParent(None)
else:
# create a new window for this plot and store it for next time
window = QtGui.QWidget()
window.resize(800, 600)
window.setWindowTitle(figname)
_plot_windows[figname] = window
# create the navigation toolbar
toolbar = NavigationToolbar(canvas, window)
# add the canvas and toolbar to the window
layout = window.layout() or QtGui.QVBoxLayout()
layout.addWidget(canvas)
layout.addWidget(toolbar)
window.setLayout(layout)
window.show()
return "[Plotted '%s']" % figname
def testEMAhandlesMissingValues(self):
window = 4
self.prices["ASX"][2] = NaN
self.prices["BHP"][0] = NaN
self.prices["CBA"][3:5] = NaN
ema = ewma(self.prices, span=window)
self.assertFalse(isnan(ema["ASX"][2]))
self.assertFalse(isnan(ema["ASX"][3]))
self.assertTrue(isnan(ema["BHP"][0]))
self.assertFalse(isnan(ema["BHP"][1]))
self.assertFalse(any(isnan(ema["CBA"][3:5])))
def testEMAhandlesMissingValues(self):
window = 4
self.prices["ASX"][2] = NaN
self.prices["BHP"][0] = NaN
self.prices["CBA"][3:5] = NaN
ema = ewma(self.prices, span = window)
self.assertFalse(isnan(ema["ASX"][2]))
self.assertFalse(isnan(ema["ASX"][3]))
self.assertTrue(isnan(ema["BHP"][0]))
self.assertFalse(isnan(ema["BHP"][1]))
self.assertFalse(any(isnan(ema["CBA"][3:5])))
def test_legacy_time_rule_arg(self):
# suppress deprecation warnings
sys.stderr = StringIO()
rng = bdate_range('1/1/2000', periods=20)
ts = Series(np.random.randn(20), index=rng)
ts = ts.take(np.random.permutation(len(ts))[:12]).sort_index()
try:
result = mom.rolling_mean(ts, 1, min_periods=1, freq='B')
expected = mom.rolling_mean(ts, 1, min_periods=1,
time_rule='WEEKDAY')
tm.assert_series_equal(result, expected)
result = mom.ewma(ts, span=5, freq='B')
expected = mom.ewma(ts, span=5, time_rule='WEEKDAY')
tm.assert_series_equal(result, expected)
finally:
sys.stderr = sys.__stderr__
def test_legacy_time_rule_arg(self):
# suppress deprecation warnings
sys.stderr = StringIO()
rng = bdate_range('1/1/2000', periods=20)
ts = Series(np.random.randn(20), index=rng)
ts = ts.take(np.random.permutation(len(ts))[:12]).sort_index()
try:
result = mom.rolling_mean(ts, 1, min_periods=1, freq='B')
expected = mom.rolling_mean(ts, 1, min_periods=1,
time_rule='WEEKDAY')
tm.assert_series_equal(result, expected)
result = mom.ewma(ts, span=5, freq='B')
expected = mom.ewma(ts, span=5, time_rule='WEEKDAY')
tm.assert_series_equal(result, expected)
finally:
sys.stderr = sys.__stderr__
def test_ewma_nan_handling(self):
s = Series([1.] + [np.nan] * 5 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [1.] * len(s))
s = Series([np.nan] * 2 + [1.] + [np.nan] * 2 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [np.nan] * 2 + [1.] * 4)
# GH 7603
s0 = Series([np.nan, 1., 101.])
s1 = Series([1., np.nan, 101.])
s2 = Series([np.nan, 1., np.nan, np.nan, 101., np.nan])
com = 2.
alpha = 1. / (1. + com)
def simple_wma(s, w):
return (s.multiply(w).cumsum() / w.cumsum()).fillna(method='ffill')
for (s, adjust, ignore_na, w) in [
(s0, True, False, [np.nan, (1.0 - alpha), 1.]),
(s0, True, True, [np.nan, (1.0 - alpha), 1.]),
(s0, False, False, [np.nan, (1.0 - alpha), alpha]),
(s0, False, True, [np.nan, (1.0 - alpha), alpha]),
(s1, True, False, [(1.0 - alpha)**2, np.nan, 1.]),
(s1, True, True, [(1.0 - alpha), np.nan, 1.]),
(s1, False, False, [(1.0 - alpha)**2, np.nan, alpha]),
(s1, False, True, [(1.0 - alpha), np.nan, alpha]),
(s2, True, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, 1., np.nan]),
(s2, True, True, [np.nan, (1.0 - alpha), np.nan, np.nan, 1., np.nan]),
(s2, False, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, alpha, np.nan]),
(s2, False, True, [np.nan, (1.0 - alpha), np.nan, np.nan, alpha, np.nan]),
]:
expected = simple_wma(s, Series(w))
result = mom.ewma(s, com=com, adjust=adjust, ignore_na=ignore_na)
assert_series_equal(result, expected)
if ignore_na is False:
# check that ignore_na defaults to False
result = mom.ewma(s, com=com, adjust=adjust)
assert_series_equal(result, expected)
def stddev_from_moving_average(timeseries):
"""
A timeseries is anomalous if the absolute value of the average of the latest
three datapoint minus the moving average is greater than three standard
deviations of the moving average. This is better for finding anomalies with
respect to the short term trends.
"""
# print('算法计算: stddev_from_moving_average')
series = pandas.Series([x[1] for x in timeseries])
expAverage = moments.ewma(series, com=50)
stdDev = moments.ewmstd(series, com=50)
return abs(series.iget(-1) - expAverage.iget(-1)) > 3 * stdDev.iget(-1)
def mean_subtraction_cumulation(timeseries):
"""
A timeseries is anomalous if the value of the next datapoint in the
series is farther than three standard deviations out in cumulative terms
after subtracting the mean from each data point.
"""
# print('算法计算: mean_subtraction_cumulation')
series = pandas.Series([x[1] if x[1] else 0 for x in timeseries])
series = series - series[0:len(series) - 1].mean()
stdDev = series[0:len(series) - 1].std()
expAverage = moments.ewma(series, com=15)
return abs(series.iget(-1)) > 3 * stdDev
def histogram(request, interval="year"):
es = pyes.ES("localhost:9200")
roll = int(request.GET.get("roll", "0"))
terms = request.GET.getlist("term")
data = dict(
(term, _get_histogram_data(es, term, interval)) for term in terms
)
frame = pandas.DataFrame(data).fillna(0)
if roll:
frame = moments.ewma(frame, roll)[roll:]
context = {
"histogram": frame,
"terms": ", ".join(terms)
}
return render_to_response("hanalytics/%s-histogram.html" % interval, context)
def process_lock(args):
cursor = db.junctions.find({"_id":args})
rain = getseriesweather('dailyrainMM',"mean","IDUBLINC2")
wind = getseriesweather('WindSpeedGustKMH',"mean","IDUBLINC2")
temperature = getseriesweather('TemperatureC',"mean","IDUBLINC2")
#json_str =json_util.dumps(cursor)
#junctions =json_util.loads(json_str)
#neighbours1 = list(db.junctions.find({"junction2.point":junctions[0]["junction1"]["point"]}))
#neighbours2 = list(db.junctions.find({"junction1.point":junctions[0]["junction2"]["point"]}))
series = []
neighbours = []
#neighbours.extend(neighbours1)
#neighbours.extend(neighbours2)
#print("1",neighbours1)
#print("2",neighbours2)
arg = args.split("/")
series1 = {"route":arg[0],"link":arg[1],"direction":arg[2]}
selected_series = getseries(series1)
#for n in neighbours:
#if not n["direction"] == series1["direction"]:
#if not (n["route"] + "/" + n["link"]) == (series1["route"] + "/" + series1["link"]):
#series.append(getseries({"route": n["route"],"link": n["link"],"direction": n["direction"]}))
shift = 10*6*24
ds = {"STT":selected_series,
"Wind":wind,
"Rain":rain,
"Temperature":temperature,
"STT1":ewma(selected_series.shift(shift), span=1+shift),
"STT2":ewma(selected_series.shift(shift), span=2+shift),
"STT3":ewma(selected_series.shift(shift), span=3+shift),
"Wind1":ewma(wind.shift(shift), span=3+shift),
"Temperature1":ewma(temperature.shift(shift), span=3+shift),
"Rain1":ewma(rain.shift(shift), span=3+shift)
}
dframe = df(ds)
t_list = list(['STT1','STT2','STT3','Rain','Wind','Temperature','Rain1','Wind1','Temperature1'])
for i,s in enumerate(dframe.columns.values):
dframe[s].fillna(method="pad", inplace=True)
train_df = dframe["2013-01-01":"2014-04-17"].resample('H', how="max",convention='end',fill_method='pad').copy()
test_df = train_df.copy()["2013-09-01":"2014-02-15"]
train_df = train_df.copy()["2013-02-16":"2014-04-17"]
train_df.to_csv("pickle/" + args.replace("/","_") + "/" + "training_data.csv")
test_df.to_csv("pickle/" + args.replace("/","_") + "/" + "testing_data.csv")
def stochasticOscillator(hlc, span): #print hlc [0:10] result = [] for i in range(0, len(hlc)): hlc_from = i-span if i>span else 0 hlc_to = i+1 if i<len(hlc) else len(hlc) try: close = hlc[i][2] lo = getLo(hlc[hlc_from:hlc_to,1]) hi = getHi(hlc[hlc_from:hlc_to][0]) #numerator = float(hi/lo) if lo!=0 else 0 k = (float((close - lo) / (hi - lo))) if (hi - lo)!=0 else 0 result.append(k) except: print hlc_from, hlc_to, i raise d = moments.ewma(np.array(result).ravel(), span=3) return d
def stochasticOscillator(hlc, span):
#print hlc [0:10]
result = []
for i in range(0, len(hlc)):
hlc_from = i - span if i > span else 0
hlc_to = i + 1 if i < len(hlc) else len(hlc)
try:
close = hlc[i][2]
lo = getLo(hlc[hlc_from:hlc_to, 1])
hi = getHi(hlc[hlc_from:hlc_to][0])
#numerator = float(hi/lo) if lo!=0 else 0
k = (float((close - lo) / (hi - lo))) if (hi - lo) != 0 else 0
result.append(k)
except:
print hlc_from, hlc_to, i
raise
d = moments.ewma(np.array(result).ravel(), span=3)
return d
def MinVarianceAllocation(history):
num_assets = len(history)
# compute covariance matrix
sigma = np.cov(history)
projected_cost = [moments.ewma(h, halflife=10)[-1] for h in history]
bounds = tuple((0.001,1) for _ in range(num_assets))
opt_results = optimize.minimize(cost_func,
x0=np.array([1.0] * num_assets),
constraints=({'type': 'eq', 'fun': lambda w: np.sum(w) - 1.0}),
bounds=bounds,
tol=np.mean(projected_cost)*0.01,
args=(projected_cost, sigma),
method='SLSQP')
return opt_results['x'], opt_results['success']
def outlier_removal_smoothing_ewma(tsData):
"""
>>> data = pd.read_csv('input.csv')
>>> outlier_removal_smoothing_ewma(data)
{'buy': 43904257, 'sale': 55054485, 'method': 'outlier_removal_smoothing_ewma'}
"""
tsData['ingestdatetime'] = tsData.apply(lambda row: datetime.strptime(str(row['ingestdate']), "%Y%m%d"), axis=1)
tsData = tsData.sort(['ingestdate'], ascending=[1])
tsData['qtyavail_ewma'] = ewma(tsData['qtyavail'], span=50)
diffs = np.diff(tsData['qtyavail_ewma'])
result_sale = abs(sum(filter(lambda x: x < 0, diffs)))
result_buy = abs(sum(filter(lambda x: x > 0, diffs)))
result = {}
result['sale'] = int(result_sale)
result['buy'] = int(result_buy)
result['method'] = inspect.stack()[0][3]
return result
def viewcompare():
assert_data()
df = session['df'].copy()
this_year = df.iloc[-1]['ts'].year
last_year = this_year - 1
this_year_df = df[df.apply(lambda x: x['ts'].year==this_year, axis=1)][
['ts','COST','USAGE']]
last_year_df = df[df.apply(lambda x: x['ts'].year==last_year, axis=1)][
['ts','COST','USAGE']]
last_year_df = last_year_df.rename(columns={'COST':'COST2',
'USAGE':'USAGE2'})
# Give last year this year's date (for comparison's sake)
last_year_df['ts'] = last_year_df['ts'].apply(
lambda x: x.replace(year=this_year))
#this_year_df = this_year_df.set_index('ts',drop=False)
#last_year_df = last_year_df.set_index('ts',drop=False)
df = this_year_df.merge(last_year_df, how='left')
df = df.set_index('ts', drop=False)
# Aggregate to daily
df = df.resample('D', how='sum')
df['ts'] = df.index
# Smoth the data
from pandas.stats.moments import ewma
cols = ['COST','COST2','USAGE','USAGE2']
df[cols] = ewma(df[cols], span=3)
cols = """
[{id: 'date', label: 'Date', type: 'datetime'},
{id: 'USAGE', label: 'Usage (%s) (kWh)', type: 'number'},
{id: 'COST', label: 'Cost (%s) ($)', type: 'number'},
{id: 'USAGE2', label: 'Usage (%s) (kWh)', type: 'number'},
{id: 'COST2', label: 'Cost (%s) ($)', type: 'number'} ]
""" % (this_year, this_year, last_year, last_year)
template = (" {c:[{v: 'Date(%(ts)s)'}, "
"{v: %(USAGE)s}, {v: %(COST)s}, "
"{v: %(USAGE2)s}, {v: %(COST2)s}]},\n")
return google_linechart(df, cols, template)
def forecast(self, forecast_start_str, forecast_period_in_days, periods_of_data_to_use):
'''Perform the forecast and return forecast as pandas Series object'''
#create forecast index
forecast_index = date_range(forecast_start_str, periods=forecast_period_in_days)
#Extract only that data which is necessary to make the first moving average calculation
data_series = self.training_ts.tail(periods_of_data_to_use)
forecast = Series()
for time in forecast_index:
#forecasted value is last value in rolling_mean list - all others are NaN because of forecast window length
if self.forecast_method == 'ma':
#Forecast using the simple moving average
forecast_value = rolling_mean(data_series, periods_of_data_to_use).loc[-1]
elif self.forecast_method == 'ewma':
#forecast using the exponentially weighted moving average
forecast_value = ewma(data_series, span=periods_of_data_to_use).loc[-1]
#print forecast_value
#remove 1-st value from data because its not needed for next forecasted value
data_series = data_series[1:]
#Append forecasted value to data because forecast is data for next iteration MA
data_series = concat([data_series, Series(forecast_value, index=[time])])
forecast = concat([forecast, Series(forecast_value, index=[time])])
return forecast
def outlier_removal_smoothing_ewma(tsData):
"""
>>> data = pd.read_csv('input.csv')
>>> outlier_removal_smoothing_ewma(data)
{'buy': 43904257, 'sale': 55054485, 'method': 'outlier_removal_smoothing_ewma'}
"""
tsData['ingestdatetime'] = tsData.apply(
lambda row: datetime.strptime(str(row['ingestdate']), "%Y%m%d"),
axis=1)
tsData = tsData.sort(['ingestdate'], ascending=[1])
tsData['qtyavail_ewma'] = ewma(tsData['qtyavail'], span=50)
diffs = np.diff(tsData['qtyavail_ewma'])
result_sale = abs(sum(filter(lambda x: x < 0, diffs)))
result_buy = abs(sum(filter(lambda x: x > 0, diffs)))
result = {}
result['sale'] = int(result_sale)
result['buy'] = int(result_buy)
result['method'] = inspect.stack()[0][3]
return result
],
how='left',
suffixes=('', i))
#zz1 = combined_df.groupby(['flight_number', 'flight_boarding_pt', 'flight_boarding_time','itemcategory',
# 'dishsubcategory','menu_cycle','destination']).size().reset_index()
#
#45 DXB 2018-05-21 08:25:00 Lunch Poultry
#zz2 = combined_df[(combined_df.flight_number == 45) & (combined_df.flight_boarding_pt == 'DXB') &
# (combined_df.flight_boarding_time == '2018-05-21 08:25:00') ]
from pandas.stats.moments import ewma
combined_df['exp_smooth'] = combined_df.groupby(
['flight_number', 'flight_boarding_pt',
'dishsubcategory'])['Meal'].apply(lambda x: ewma(x, span=15))
combined_df['exp_smooth'] = combined_df.groupby(
['flight_number', 'flight_boarding_pt', 'itemcategory',
'dishsubcategory'])['exp_smooth'].shift(7)
def holt_winters_second_order_ewma(x, span, beta):
N = x.size
alpha = 2.0 / (1 + span)
s = np.zeros((N, ))
b = np.zeros((N, ))
s[0] = x[0]
for i in range(1, N):
s[i] = alpha * x[i] + (1 - alpha) * (s[i - 1] + b[i - 1])
b[i] = beta * (s[i] - s[i - 1]) + (1 - beta) * b[i - 1]
def execute(self, strategy):
prices = strategy.get_indicator_prices()
fast_ema = ewma(prices, span = self.fast)
slow_ema = ewma(prices, span = self.slow)
levels = fast_ema > slow_ema
return Indicator(levels.astype('str'))
def getMACDAndSignalLine(pricesData, n1, n2, n3):
ema_n1 = getEMA(pricesData, n1)
ema_n2 = getEMA(pricesData, n2)
macd = ema_n1 - ema_n2
signalLine = pd.Series(mt.ewma(macd, span=n3), index=pricesData.index)
return macd, signalLine
def ema(s, n, wilder=False):
span = n if not wilder else 2 * n - 1
return moments.ewma(s, span=span)
def ema(s, n, wilder=False):
span = n if not wilder else 2*n - 1
return moments.ewma(s, span=span)
def _ema(s, n, wilder=False):
"""
https://github.com/FreddieWitherden/ta/blob/master/ta.py
"""
span = n if not wilder else 2*n - 1
return moments.ewma(s, span=span)
def calc_ewma(self, span=5):
column = 'ewma' + str(span)
self.stock[column] = ewma(self.close, span=span)
return self.stock
def execute(self, strategy):
prices = strategy.get_indicator_prices()
fast_ema = ewma(prices, span=self.fast)
slow_ema = ewma(prices, span=self.slow)
levels = fast_ema > slow_ema
return Indicator(levels.astype('str'))
def test_ewma_nan_handling(self):
s = Series([1.] + [np.nan] * 5 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [1] * len(s))
import matplotlib.pyplot as plt import pandas.util.testing as t import pandas.stats.moments as m t.N = 200 s = t.makeTimeSeries().cumsum() plt.figure(figsize=(10, 5)) plt.plot(s.index, s.values) plt.plot(s.index, m.ewma(s, 20, min_periods=1).values) f = plt.gcf() f.autofmt_xdate() plt.show()
def getEMA(pricesData, n=12):
ema = pd.Series(mt.ewma(pricesData, span=n), index=pricesData.index)
return ema
def test_ewma_nan_handling(self):
s = Series([1.] + [np.nan] * 5 + [1.])
result = mom.ewma(s, com=5)
assert_almost_equal(result, [1] * len(s))
def plot_scatter(data, ax, text, color, span=10, method='emwa'):
from pandas.stats.moments import ewma
data = ewma(data,span=span)
ax.plot(data, color=color, linewidth=1, label=text)
import numpy as np import matplotlib.pyplot as plt import pandas.stats.moments as moments import parsetmy import getwunderground typical = parsetmy.load() current = getwunderground.load() fig = plt.figure() ax1 = fig.add_subplot(111) #ax1.plot(r1.date_mmddyyyy, r1.dewpoint_c, 'o-') ax1.plot(typical.date_mmddyyyy, moments.ewma(typical.dewpoint_c,span=24*14), 'o-') ax1.plot(current['datetime'], moments.ewma(current['dewptm'],span=24), 'o-') #ax2 = fig.add_subplot(212) #ax2.plot(r2.date_mmddyyyy, r2.dewpoint_c, 'o-') fig.autofmt_xdate() plt.show()
except:
print EWMA10
raise
#V_MA.append(x_mat[i][6])
#arr_concat = np.array([[x_mat[i,c_ind]], [x_mat[i+1,c_ind]]])
#KALMAN.append(arr_concat)
#cycle, trend = stats_filters.hpfilter(HP_FILTER, lamb=1600)
#sys.exit(0)
#X_SHARPE = getSharpe(np.array(X_SHARPE).ravel(), 3)
#X_SHARPE2 = getSharpe(np.array(X_SHARPE2).ravel(), 10)
SO = stochasticOscillator(np.array(SO), 3)
#SO2 = stochasticOscillator(np.array(SO2),10)
CHANGE_U = moments.ewma(np.array(CHANGE_U).ravel(), span=3)
CHANGE_D = moments.ewma(np.array(CHANGE_D).ravel(), span=3)
#TP_SMA = moments.rolling_mean(np.array(TP),window=20,min_periods=0)
#KALMAN = np.array(KALMAN)
#KALMAN.reshape(len(KALMAN[:,0:1]), len(KALMAN[1:,:]))
#CHANGE_U = moments.rolling_mean(np.array(CHANGE_U).ravel(), window=5, min_periods=0)
#CHANGE_D = moments.rolling_mean(np.array(CHANGE_D).ravel(), window=5, min_periods=0)
#ACC_DIST1 = moments.ewma(np.array(ACC_DIST).ravel(), span=3)
#ACC_DIST2 = moments.ewma(np.array(ACC_DIST).ravel(), span=5)
#print SO[0:10]
#print X_SHARPE[20:30]
#print X_SHARPE2[20:30]
#print len(X_MA)
#print len(X_MA2)
except: print EWMA10 raise #V_MA.append(x_mat[i][6]) #arr_concat = np.array([[x_mat[i,c_ind]], [x_mat[i+1,c_ind]]]) #KALMAN.append(arr_concat) #cycle, trend = stats_filters.hpfilter(HP_FILTER, lamb=1600) #sys.exit(0) #X_SHARPE = getSharpe(np.array(X_SHARPE).ravel(), 3) #X_SHARPE2 = getSharpe(np.array(X_SHARPE2).ravel(), 10) SO = stochasticOscillator(np.array(SO),3) #SO2 = stochasticOscillator(np.array(SO2),10) CHANGE_U = moments.ewma(np.array(CHANGE_U).ravel(), span=3) CHANGE_D = moments.ewma(np.array(CHANGE_D).ravel(), span=3) #TP_SMA = moments.rolling_mean(np.array(TP),window=20,min_periods=0) #KALMAN = np.array(KALMAN) #KALMAN.reshape(len(KALMAN[:,0:1]), len(KALMAN[1:,:])) #CHANGE_U = moments.rolling_mean(np.array(CHANGE_U).ravel(), window=5, min_periods=0) #CHANGE_D = moments.rolling_mean(np.array(CHANGE_D).ravel(), window=5, min_periods=0) #ACC_DIST1 = moments.ewma(np.array(ACC_DIST).ravel(), span=3) #ACC_DIST2 = moments.ewma(np.array(ACC_DIST).ravel(), span=5) #print SO[0:10] #print X_SHARPE[20:30] #print X_SHARPE2[20:30] #print len(X_MA)
def getMACDAndSignalLine(pricesData, n1, n2, n3):
ema_n1 = getEMA(pricesData, n1)
ema_n2 = getEMA(pricesData, n2)
macd = ema_n1 - ema_n2
signalLine = pd.Series(mt.ewma(macd, span=n3), index=pricesData.index)
return macd, signalLine
def calc_ewma(self, span=5):
column = 'ewma' + str(span)
self.stock[column] = ewma(self.close, span=span)
return self.stock
