def test_ewma_span_com_args(self): A = moments.ewma(self.arr, com=9.5) B = moments.ewma(self.arr, span=20) assert_almost_equal(A, B) self.assertRaises(Exception, moments.ewma, self.arr, com=9.5, span=20) self.assertRaises(Exception, moments.ewma, self.arr)
def test_ewma_span_com_args(self): A = mom.ewma(self.arr, com=9.5) B = mom.ewma(self.arr, span=20) assert_almost_equal(A, B) self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, span=20) self.assertRaises(Exception, mom.ewma, self.arr)
def test_ewma_halflife_arg(self): A = mom.ewma(self.arr, com=13.932726172912965) B = mom.ewma(self.arr, halflife=10.0) assert_almost_equal(A, B) self.assertRaises(Exception, mom.ewma, self.arr, span=20, halflife=50) self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, halflife=50) self.assertRaises(Exception, mom.ewma, self.arr, com=9.5, span=20, halflife=50) self.assertRaises(Exception, mom.ewma, self.arr)
def compute_macd(data): data[’ewma5’] = ewma(data.close, span=5) data[’ewma35’] = ewma(data.close, span=35) data[’macd’] = data.ewma5 - data.ewma35 data[’macd_signal’] = ewma(data.macd, span=5) data[’histogram’] = data.macd - data.macd_signal
def moving_average_convergence(x, nslow=50, nfast=10): """ compute the crossover (Moving Average Convergence/Divergence) using a fast and slow exponential moving avg' return value is emaslow, emafast, crossover which are len(x) arrays """ emaslow = mom.ewma(x, span=nslow) emafast = mom.ewma(x, span=nfast) # emaslow = moving_average(x, nslow, type='exponential') # emafast = moving_average(x, nfast, type='exponential') # print x # time.sleep(10) return emaslow, emafast, emafast - emaslow
def calc_rtg_daily(daily_df, horizon): print "Caculating daily rtg..." result_df = filter_expandable(daily_df) print "Calculating rtg0..." halflife = horizon / 2 # result_df['dk'] = np.exp( -1.0 * halflife * (result_df['gdate'] - result_df['last']).astype('timedelta64[D]').astype(int) ) result_df['cum_ret'] = pd.rolling_sum(result_df['log_ret'], horizon) result_df['sum'] = result_df['mean'] * result_df['count'] result_df['det_diff'] = result_df['sum'].diff() result_df['det_diff_dk'] = ewma(result_df['det_diff'], halflife=horizon ) result_df['rtg0'] = result_df['det_diff_dk'] * result_df['det_diff_dk'] # result_df['median'] = -1.0 * (result_df['median'] - 3) # result_df['med_diff'] = result_df['median'].unstack().diff().stack() # result_df['med_diff_dk'] = pd.rolling_sum( result_df['dk'] * result_df['med_diff'], window=horizon ) # result_df['rtg0'] = (np.sign(result_df['med_diff_dk']) * np.sign(result_df['cum_ret'])).clip(lower=0) * result_df['med_diff_dk'] demean = lambda x: (x - x.mean()) indgroups = result_df[['rtg0', 'gdate', 'ind1']].groupby(['gdate', 'ind1'], sort=True).transform(demean) result_df['rtg0_ma'] = indgroups['rtg0'] # result_df['rtg0_ma'] = result_df['rtg0_ma'] * (np.sign(result_df['rtg0_ma']) * np.sign(result_df['cum_ret'])) # result_df['rtg0_ma'] = result_df['rtg0'] shift_df = result_df.unstack().shift(1).stack() result_df['rtg1_ma'] = shift_df['rtg0_ma'] return result_df
def exp_smoothing(df): ''' Function to calculate exponential smoothing Pandas functions are used to calculate moving average and standard deviation Upper and lower thresholds are calculated by Upper threshold = moving average + standard deviation Lower threshold = moving average - standard deviation ''' #Calculate exp weighted moving average ExpWeightedMovingAvg = ewma(df,span=N) ExpWeightedMovingAvg.columns=['Exp Weighted Moving Avg'] #Calculate exp weighted moving std ExpWeightedMovingStd = ewmstd(df,span=N) ExpWeightedMovingStd.columns=['Std Deviation'] s1=df['Original data'] s2=ExpWeightedMovingAvg['Exp Weighted Moving Avg'] s3=ExpWeightedMovingStd['Std Deviation'] s4=s2.add(s3, fill_value=0) s5=s2.sub(s3, fill_value=0) df2=DataFrame(s1,columns=['Original data']) df2['Exp Weighted Moving Avg']=s2 df2['Std Deviation']=s3 df2['Upper Threshold']=s4 df2['Lower Threshold']=s5 return df2
def exp_smoothing(df): ''' Function to calculate exponential smoothing Pandas functions are used to calculate moving average and standard deviation Upper and lower thresholds are calculated by Upper threshold = moving average + standard deviation Lower threshold = moving average - standard deviation ''' #Calculate exp weighted moving average ExpWeightedMovingAvg = ewma(df, span=N) ExpWeightedMovingAvg.columns = ['Exp Weighted Moving Avg'] #Calculate exp weighted moving std ExpWeightedMovingStd = ewmstd(df, span=N) ExpWeightedMovingStd.columns = ['Std Deviation'] s1 = df['Original data'] s2 = ExpWeightedMovingAvg['Exp Weighted Moving Avg'] s3 = ExpWeightedMovingStd['Std Deviation'] s4 = s2.add(s3, fill_value=0) s5 = s2.sub(s3, fill_value=0) df2 = DataFrame(s1, columns=['Original data']) df2['Exp Weighted Moving Avg'] = s2 df2['Std Deviation'] = s3 df2['Upper Threshold'] = s4 df2['Lower Threshold'] = s5 return df2
def pd_ema(arg, ratio=0.1): ''' EMA, implemented with `pandas.stats.moments.ewma` ''' span = 2.0 / (1-ratio) - 1 arg = utils.safe_series(arg) rval = ewma(arg, span=span) utils.safe_name(rval, name='pdEMA') return rval
def test_ewma(self): self._check_ew(mom.ewma) arr = np.zeros(1000) arr[5] = 1 result = mom.ewma(arr, span=100, adjust=False).sum() self.assert_(np.abs(result - 1) < 1e-2)
def ewma(params, df): ''' Ewmas specified columns of the df ''' print('Ewma data') #unknownFeatures = features['unknowns'] #knownFeatures = features['knowns'] dfX, dfy = df2xy(df, params.data_cols.features, params.data_cols.target_col) resultDf = pd.DataFrame() for col in dfX.columns: series = dfX[col] # take EWMA in both directions with a smaller span term fwd = moments.ewma(series, span=params.hypers.days_to_average) # take EWMA in fwd direction #bwd = ewma( series[::-1], span=self.daysToAverage ) # take EWMA in bwd direction #c = np.vstack(( fwd, bwd[::-1] )) # lump fwd and bwd together #c = np.mean( c, axis=0 ) # average c = fwd cDf = pd.DataFrame(c,columns=[col]) newDf = pd.concat([resultDf, cDf], axis=1, ignore_index=True) columns = list(resultDf.columns)+[col] newDf.columns = columns resultDf = newDf # Attach ewma'd Xs with y and send back df dfX = resultDf.round(1) df = xy2df(dfX, dfy) return df
def mpl_plot_ewma_embedded(figname, xs, ys, span): # create the figure and axes for the plot fig = Figure(figsize=(8, 6), dpi=75, facecolor=(1, 1, 1), edgecolor=(0, 0, 0)) ax = fig.add_subplot(111) # calculate the moving average ewma_ys = ewma(ys, span=span) # plot the data ax.plot(xs, ys, alpha=0.4, label="Raw") ax.plot(xs, ewma_ys, label="EWMA") ax.legend() # write the figure to a temporary image file filename = os.path.join(os.environ["TEMP"], "xlplot_%s.png" % figname) canvas = FigureCanvas(fig) canvas.draw() canvas.print_png(filename) # Show the figure in Excel as a Picture object on the same sheet # the function is being called from. xl = xl_app() caller = xlfCaller() sheet = xl.Range(caller.address).Worksheet # if a picture with the same figname already exists then get the position # and size from the old picture and delete it. for old_picture in sheet.Pictures(): if old_picture.Name == figname: height = old_picture.Height width = old_picture.Width top = old_picture.Top left = old_picture.Left old_picture.Delete() break else: # otherwise place the picture below the calling cell. top_left = sheet.Cells(caller.rect.last_row+2, caller.rect.last_col+1) top = top_left.Top left = top_left.Left width, height = fig.bbox.bounds[2:] # insert the picture # Ref: http://msdn.microsoft.com/en-us/library/office/ff198302%28v=office.15%29.aspx picture = sheet.Shapes.AddPicture(Filename=filename, LinkToFile=0, # msoFalse SaveWithDocument=-1, # msoTrue Left=left, Top=top, Width=width, Height=height) # set the name of the new picture so we can find it next time picture.Name = figname # delete the temporary file os.unlink(filename) return "[Plotted '%s']" % figname
def simpleMApredict(x,span,periods = pred_period): x_predict = np.zeros((span+periods,)) x_predict[:span] = x[-span:] pred = ewma(x_predict,span)[span:] pred = pred.round() pred[pred < 0] = 0 return pred
def predict(x, span, periods=pred_period): x_predict = np.zeros((span + periods, )) x_predict[:span] = x[-span:] pred = ewma(x_predict, span)[span:] pred = pred.round() pred[pred < 0] = 0 return pred
def predict_EWMA(x,span=3,periods = pred_period): x_predict = np.zeros((span+periods,)) x_predict[:span] = x[-span:] pred = ewma(x_predict,span)[span:] pred = pred.round() pred[pred < 0] = 0 return pred
def McClellanOscillator(): """ McClellan Oscillator http://stockcharts.com/school/doku.php?id=chart_school:technical_indicators:mcclellan_oscillator """ def RANA(): """ Ratio Adjusted Net Advances (RANA) Returns RANA for whole dataset """ return ((data['Advances'] - data['Declines']) / (data['Advances'] + data['Declines']) * 1000) ####### Calculate RANA rana = RANA() ####### Calculate EMAS ema19 = ewma(rana, span=19) ema39 = ewma(rana, span=39) return ema19 - ema39
def mpl_plot_ewma(figname, xs, ys, span): """ Show a matplotlib line plot of xs vs ys and ewma(ys, span) in an interactive window. :param figname: name to use for this plot's window :param xs: list of x values as a column :param ys: list of y values as a column :param span: ewma span """ # Get the Qt app. # Note: no need to 'exec' this as it will be polled in the main windows loop. app = get_qt_app() # create the figure and axes for the plot fig = Figure(figsize=(600, 600), dpi=72, facecolor=(1, 1, 1), edgecolor=(0, 0, 0)) ax = fig.add_subplot(111) # calculate the moving average ewma_ys = ewma(ys, span=span) # plot the data ax.plot(xs, ys, alpha=0.4, label="Raw") ax.plot(xs, ewma_ys, label="EWMA") ax.legend() # generate the canvas to display the plot canvas = FigureCanvas(fig) # Get or create the Qt windows to show the chart in. if figname in _plot_windows: # get from the global dict and clear any previous widgets window = _plot_windows[figname] layout = window.layout() if layout: for i in reversed(range(layout.count())): layout.itemAt(i).widget().setParent(None) else: # create a new window for this plot and store it for next time window = QtGui.QWidget() window.resize(800, 600) window.setWindowTitle(figname) _plot_windows[figname] = window # create the navigation toolbar toolbar = NavigationToolbar(canvas, window) # add the canvas and toolbar to the window layout = window.layout() or QtGui.QVBoxLayout() layout.addWidget(canvas) layout.addWidget(toolbar) window.setLayout(layout) window.show() return "[Plotted '%s']" % figname
def test_ewma(self): self._check_ew(mom.ewma) arr = np.zeros(1000) arr[5] = 1 result = mom.ewma(arr, span=100, adjust=False).sum() self.assertTrue(np.abs(result - 1) < 1e-2) s = Series([1.0, 2.0, 4.0, 8.0]) expected = Series([1.0, 1.6, 2.736842, 4.923077]) for f in [ lambda s: mom.ewma(s, com=2.0, adjust=True), lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=False), lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=True), ]: result = f(s) assert_series_equal(result, expected) expected = Series([1.0, 1.333333, 2.222222, 4.148148]) for f in [ lambda s: mom.ewma(s, com=2.0, adjust=False), lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=False), lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=True), ]: result = f(s) assert_series_equal(result, expected)
def test_ewma(self): self._check_ew(mom.ewma) arr = np.zeros(1000) arr[5] = 1 result = mom.ewma(arr, span=100, adjust=False).sum() self.assertTrue(np.abs(result - 1) < 1e-2) s = Series([1.0, 2.0, 4.0, 8.0]) expected = Series([1.0, 1.6, 2.736842, 4.923077]) for f in [lambda s: mom.ewma(s, com=2.0, adjust=True), lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=False), lambda s: mom.ewma(s, com=2.0, adjust=True, ignore_na=True), ]: result = f(s) assert_series_equal(result, expected) expected = Series([1.0, 1.333333, 2.222222, 4.148148]) for f in [lambda s: mom.ewma(s, com=2.0, adjust=False), lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=False), lambda s: mom.ewma(s, com=2.0, adjust=False, ignore_na=True), ]: result = f(s) assert_series_equal(result, expected)
def test_ewma_nan_handling(self): s = Series([1.] + [np.nan] * 5 + [1.]) result = mom.ewma(s, com=5) assert_almost_equal(result, [1.] * len(s)) s = Series([np.nan] * 2 + [1.] + [np.nan] * 2 + [1.]) result = mom.ewma(s, com=5) assert_almost_equal(result, [np.nan] * 2 + [1.] * 4) # GH 7603 s0 = Series([np.nan, 1., 101.]) s1 = Series([1., np.nan, 101.]) s2 = Series([np.nan, 1., np.nan, np.nan, 101., np.nan]) com = 2. alpha = 1. / (1. + com) def simple_wma(s, w): return (s.multiply(w).cumsum() / w.cumsum()).fillna(method='ffill') for (s, adjust, ignore_na, w) in [ (s0, True, False, [np.nan, (1.0 - alpha), 1.]), (s0, True, True, [np.nan, (1.0 - alpha), 1.]), (s0, False, False, [np.nan, (1.0 - alpha), alpha]), (s0, False, True, [np.nan, (1.0 - alpha), alpha]), (s1, True, False, [(1.0 - alpha)**2, np.nan, 1.]), (s1, True, True, [(1.0 - alpha), np.nan, 1.]), (s1, False, False, [(1.0 - alpha)**2, np.nan, alpha]), (s1, False, True, [(1.0 - alpha), np.nan, alpha]), (s2, True, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, 1., np.nan]), (s2, True, True, [np.nan, (1.0 - alpha), np.nan, np.nan, 1., np.nan]), (s2, False, False, [np.nan, (1.0 - alpha)**3, np.nan, np.nan, alpha, np.nan]), (s2, False, True, [np.nan, (1.0 - alpha), np.nan, np.nan, alpha, np.nan]), ]: expected = simple_wma(s, Series(w)) result = mom.ewma(s, com=com, adjust=adjust, ignore_na=ignore_na) assert_series_equal(result, expected) if ignore_na is False: # check that ignore_na defaults to False result = mom.ewma(s, com=com, adjust=adjust) assert_series_equal(result, expected)
def testEMAhandlesMissingValues(self): window = 4 self.prices["ASX"][2] = NaN self.prices["BHP"][0] = NaN self.prices["CBA"][3:5] = NaN ema = ewma(self.prices, span=window) self.assertFalse(isnan(ema["ASX"][2])) self.assertFalse(isnan(ema["ASX"][3])) self.assertTrue(isnan(ema["BHP"][0])) self.assertFalse(isnan(ema["BHP"][1])) self.assertFalse(any(isnan(ema["CBA"][3:5])))
def testEMAhandlesMissingValues(self): window = 4 self.prices["ASX"][2] = NaN self.prices["BHP"][0] = NaN self.prices["CBA"][3:5] = NaN ema = ewma(self.prices, span = window) self.assertFalse(isnan(ema["ASX"][2])) self.assertFalse(isnan(ema["ASX"][3])) self.assertTrue(isnan(ema["BHP"][0])) self.assertFalse(isnan(ema["BHP"][1])) self.assertFalse(any(isnan(ema["CBA"][3:5])))
def test_legacy_time_rule_arg(self): # suppress deprecation warnings sys.stderr = StringIO() rng = bdate_range('1/1/2000', periods=20) ts = Series(np.random.randn(20), index=rng) ts = ts.take(np.random.permutation(len(ts))[:12]).sort_index() try: result = mom.rolling_mean(ts, 1, min_periods=1, freq='B') expected = mom.rolling_mean(ts, 1, min_periods=1, time_rule='WEEKDAY') tm.assert_series_equal(result, expected) result = mom.ewma(ts, span=5, freq='B') expected = mom.ewma(ts, span=5, time_rule='WEEKDAY') tm.assert_series_equal(result, expected) finally: sys.stderr = sys.__stderr__
def stddev_from_moving_average(timeseries): """ A timeseries is anomalous if the absolute value of the average of the latest three datapoint minus the moving average is greater than three standard deviations of the moving average. This is better for finding anomalies with respect to the short term trends. """ # print('算法计算: stddev_from_moving_average') series = pandas.Series([x[1] for x in timeseries]) expAverage = moments.ewma(series, com=50) stdDev = moments.ewmstd(series, com=50) return abs(series.iget(-1) - expAverage.iget(-1)) > 3 * stdDev.iget(-1)
def mean_subtraction_cumulation(timeseries): """ A timeseries is anomalous if the value of the next datapoint in the series is farther than three standard deviations out in cumulative terms after subtracting the mean from each data point. """ # print('算法计算: mean_subtraction_cumulation') series = pandas.Series([x[1] if x[1] else 0 for x in timeseries]) series = series - series[0:len(series) - 1].mean() stdDev = series[0:len(series) - 1].std() expAverage = moments.ewma(series, com=15) return abs(series.iget(-1)) > 3 * stdDev
def histogram(request, interval="year"): es = pyes.ES("localhost:9200") roll = int(request.GET.get("roll", "0")) terms = request.GET.getlist("term") data = dict( (term, _get_histogram_data(es, term, interval)) for term in terms ) frame = pandas.DataFrame(data).fillna(0) if roll: frame = moments.ewma(frame, roll)[roll:] context = { "histogram": frame, "terms": ", ".join(terms) } return render_to_response("hanalytics/%s-histogram.html" % interval, context)
def process_lock(args): cursor = db.junctions.find({"_id":args}) rain = getseriesweather('dailyrainMM',"mean","IDUBLINC2") wind = getseriesweather('WindSpeedGustKMH',"mean","IDUBLINC2") temperature = getseriesweather('TemperatureC',"mean","IDUBLINC2") #json_str =json_util.dumps(cursor) #junctions =json_util.loads(json_str) #neighbours1 = list(db.junctions.find({"junction2.point":junctions[0]["junction1"]["point"]})) #neighbours2 = list(db.junctions.find({"junction1.point":junctions[0]["junction2"]["point"]})) series = [] neighbours = [] #neighbours.extend(neighbours1) #neighbours.extend(neighbours2) #print("1",neighbours1) #print("2",neighbours2) arg = args.split("/") series1 = {"route":arg[0],"link":arg[1],"direction":arg[2]} selected_series = getseries(series1) #for n in neighbours: #if not n["direction"] == series1["direction"]: #if not (n["route"] + "/" + n["link"]) == (series1["route"] + "/" + series1["link"]): #series.append(getseries({"route": n["route"],"link": n["link"],"direction": n["direction"]})) shift = 10*6*24 ds = {"STT":selected_series, "Wind":wind, "Rain":rain, "Temperature":temperature, "STT1":ewma(selected_series.shift(shift), span=1+shift), "STT2":ewma(selected_series.shift(shift), span=2+shift), "STT3":ewma(selected_series.shift(shift), span=3+shift), "Wind1":ewma(wind.shift(shift), span=3+shift), "Temperature1":ewma(temperature.shift(shift), span=3+shift), "Rain1":ewma(rain.shift(shift), span=3+shift) } dframe = df(ds) t_list = list(['STT1','STT2','STT3','Rain','Wind','Temperature','Rain1','Wind1','Temperature1']) for i,s in enumerate(dframe.columns.values): dframe[s].fillna(method="pad", inplace=True) train_df = dframe["2013-01-01":"2014-04-17"].resample('H', how="max",convention='end',fill_method='pad').copy() test_df = train_df.copy()["2013-09-01":"2014-02-15"] train_df = train_df.copy()["2013-02-16":"2014-04-17"] train_df.to_csv("pickle/" + args.replace("/","_") + "/" + "training_data.csv") test_df.to_csv("pickle/" + args.replace("/","_") + "/" + "testing_data.csv")
def stochasticOscillator(hlc, span): #print hlc [0:10] result = [] for i in range(0, len(hlc)): hlc_from = i-span if i>span else 0 hlc_to = i+1 if i<len(hlc) else len(hlc) try: close = hlc[i][2] lo = getLo(hlc[hlc_from:hlc_to,1]) hi = getHi(hlc[hlc_from:hlc_to][0]) #numerator = float(hi/lo) if lo!=0 else 0 k = (float((close - lo) / (hi - lo))) if (hi - lo)!=0 else 0 result.append(k) except: print hlc_from, hlc_to, i raise d = moments.ewma(np.array(result).ravel(), span=3) return d
def stochasticOscillator(hlc, span): #print hlc [0:10] result = [] for i in range(0, len(hlc)): hlc_from = i - span if i > span else 0 hlc_to = i + 1 if i < len(hlc) else len(hlc) try: close = hlc[i][2] lo = getLo(hlc[hlc_from:hlc_to, 1]) hi = getHi(hlc[hlc_from:hlc_to][0]) #numerator = float(hi/lo) if lo!=0 else 0 k = (float((close - lo) / (hi - lo))) if (hi - lo) != 0 else 0 result.append(k) except: print hlc_from, hlc_to, i raise d = moments.ewma(np.array(result).ravel(), span=3) return d
def MinVarianceAllocation(history): num_assets = len(history) # compute covariance matrix sigma = np.cov(history) projected_cost = [moments.ewma(h, halflife=10)[-1] for h in history] bounds = tuple((0.001,1) for _ in range(num_assets)) opt_results = optimize.minimize(cost_func, x0=np.array([1.0] * num_assets), constraints=({'type': 'eq', 'fun': lambda w: np.sum(w) - 1.0}), bounds=bounds, tol=np.mean(projected_cost)*0.01, args=(projected_cost, sigma), method='SLSQP') return opt_results['x'], opt_results['success']
def outlier_removal_smoothing_ewma(tsData): """ >>> data = pd.read_csv('input.csv') >>> outlier_removal_smoothing_ewma(data) {'buy': 43904257, 'sale': 55054485, 'method': 'outlier_removal_smoothing_ewma'} """ tsData['ingestdatetime'] = tsData.apply(lambda row: datetime.strptime(str(row['ingestdate']), "%Y%m%d"), axis=1) tsData = tsData.sort(['ingestdate'], ascending=[1]) tsData['qtyavail_ewma'] = ewma(tsData['qtyavail'], span=50) diffs = np.diff(tsData['qtyavail_ewma']) result_sale = abs(sum(filter(lambda x: x < 0, diffs))) result_buy = abs(sum(filter(lambda x: x > 0, diffs))) result = {} result['sale'] = int(result_sale) result['buy'] = int(result_buy) result['method'] = inspect.stack()[0][3] return result
def viewcompare(): assert_data() df = session['df'].copy() this_year = df.iloc[-1]['ts'].year last_year = this_year - 1 this_year_df = df[df.apply(lambda x: x['ts'].year==this_year, axis=1)][ ['ts','COST','USAGE']] last_year_df = df[df.apply(lambda x: x['ts'].year==last_year, axis=1)][ ['ts','COST','USAGE']] last_year_df = last_year_df.rename(columns={'COST':'COST2', 'USAGE':'USAGE2'}) # Give last year this year's date (for comparison's sake) last_year_df['ts'] = last_year_df['ts'].apply( lambda x: x.replace(year=this_year)) #this_year_df = this_year_df.set_index('ts',drop=False) #last_year_df = last_year_df.set_index('ts',drop=False) df = this_year_df.merge(last_year_df, how='left') df = df.set_index('ts', drop=False) # Aggregate to daily df = df.resample('D', how='sum') df['ts'] = df.index # Smoth the data from pandas.stats.moments import ewma cols = ['COST','COST2','USAGE','USAGE2'] df[cols] = ewma(df[cols], span=3) cols = """ [{id: 'date', label: 'Date', type: 'datetime'}, {id: 'USAGE', label: 'Usage (%s) (kWh)', type: 'number'}, {id: 'COST', label: 'Cost (%s) ($)', type: 'number'}, {id: 'USAGE2', label: 'Usage (%s) (kWh)', type: 'number'}, {id: 'COST2', label: 'Cost (%s) ($)', type: 'number'} ] """ % (this_year, this_year, last_year, last_year) template = (" {c:[{v: 'Date(%(ts)s)'}, " "{v: %(USAGE)s}, {v: %(COST)s}, " "{v: %(USAGE2)s}, {v: %(COST2)s}]},\n") return google_linechart(df, cols, template)
def forecast(self, forecast_start_str, forecast_period_in_days, periods_of_data_to_use): '''Perform the forecast and return forecast as pandas Series object''' #create forecast index forecast_index = date_range(forecast_start_str, periods=forecast_period_in_days) #Extract only that data which is necessary to make the first moving average calculation data_series = self.training_ts.tail(periods_of_data_to_use) forecast = Series() for time in forecast_index: #forecasted value is last value in rolling_mean list - all others are NaN because of forecast window length if self.forecast_method == 'ma': #Forecast using the simple moving average forecast_value = rolling_mean(data_series, periods_of_data_to_use).loc[-1] elif self.forecast_method == 'ewma': #forecast using the exponentially weighted moving average forecast_value = ewma(data_series, span=periods_of_data_to_use).loc[-1] #print forecast_value #remove 1-st value from data because its not needed for next forecasted value data_series = data_series[1:] #Append forecasted value to data because forecast is data for next iteration MA data_series = concat([data_series, Series(forecast_value, index=[time])]) forecast = concat([forecast, Series(forecast_value, index=[time])]) return forecast
def outlier_removal_smoothing_ewma(tsData): """ >>> data = pd.read_csv('input.csv') >>> outlier_removal_smoothing_ewma(data) {'buy': 43904257, 'sale': 55054485, 'method': 'outlier_removal_smoothing_ewma'} """ tsData['ingestdatetime'] = tsData.apply( lambda row: datetime.strptime(str(row['ingestdate']), "%Y%m%d"), axis=1) tsData = tsData.sort(['ingestdate'], ascending=[1]) tsData['qtyavail_ewma'] = ewma(tsData['qtyavail'], span=50) diffs = np.diff(tsData['qtyavail_ewma']) result_sale = abs(sum(filter(lambda x: x < 0, diffs))) result_buy = abs(sum(filter(lambda x: x > 0, diffs))) result = {} result['sale'] = int(result_sale) result['buy'] = int(result_buy) result['method'] = inspect.stack()[0][3] return result
], how='left', suffixes=('', i)) #zz1 = combined_df.groupby(['flight_number', 'flight_boarding_pt', 'flight_boarding_time','itemcategory', # 'dishsubcategory','menu_cycle','destination']).size().reset_index() # #45 DXB 2018-05-21 08:25:00 Lunch Poultry #zz2 = combined_df[(combined_df.flight_number == 45) & (combined_df.flight_boarding_pt == 'DXB') & # (combined_df.flight_boarding_time == '2018-05-21 08:25:00') ] from pandas.stats.moments import ewma combined_df['exp_smooth'] = combined_df.groupby( ['flight_number', 'flight_boarding_pt', 'dishsubcategory'])['Meal'].apply(lambda x: ewma(x, span=15)) combined_df['exp_smooth'] = combined_df.groupby( ['flight_number', 'flight_boarding_pt', 'itemcategory', 'dishsubcategory'])['exp_smooth'].shift(7) def holt_winters_second_order_ewma(x, span, beta): N = x.size alpha = 2.0 / (1 + span) s = np.zeros((N, )) b = np.zeros((N, )) s[0] = x[0] for i in range(1, N): s[i] = alpha * x[i] + (1 - alpha) * (s[i - 1] + b[i - 1]) b[i] = beta * (s[i] - s[i - 1]) + (1 - beta) * b[i - 1]
def execute(self, strategy): prices = strategy.get_indicator_prices() fast_ema = ewma(prices, span = self.fast) slow_ema = ewma(prices, span = self.slow) levels = fast_ema > slow_ema return Indicator(levels.astype('str'))
def getMACDAndSignalLine(pricesData, n1, n2, n3): ema_n1 = getEMA(pricesData, n1) ema_n2 = getEMA(pricesData, n2) macd = ema_n1 - ema_n2 signalLine = pd.Series(mt.ewma(macd, span=n3), index=pricesData.index) return macd, signalLine
def ema(s, n, wilder=False): span = n if not wilder else 2 * n - 1 return moments.ewma(s, span=span)
def ema(s, n, wilder=False): span = n if not wilder else 2*n - 1 return moments.ewma(s, span=span)
def _ema(s, n, wilder=False): """ https://github.com/FreddieWitherden/ta/blob/master/ta.py """ span = n if not wilder else 2*n - 1 return moments.ewma(s, span=span)
def calc_ewma(self, span=5): column = 'ewma' + str(span) self.stock[column] = ewma(self.close, span=span) return self.stock
def execute(self, strategy): prices = strategy.get_indicator_prices() fast_ema = ewma(prices, span=self.fast) slow_ema = ewma(prices, span=self.slow) levels = fast_ema > slow_ema return Indicator(levels.astype('str'))
def test_ewma_nan_handling(self): s = Series([1.] + [np.nan] * 5 + [1.]) result = mom.ewma(s, com=5) assert_almost_equal(result, [1] * len(s))
import matplotlib.pyplot as plt import pandas.util.testing as t import pandas.stats.moments as m t.N = 200 s = t.makeTimeSeries().cumsum() plt.figure(figsize=(10, 5)) plt.plot(s.index, s.values) plt.plot(s.index, m.ewma(s, 20, min_periods=1).values) f = plt.gcf() f.autofmt_xdate() plt.show()
def getEMA(pricesData, n=12): ema = pd.Series(mt.ewma(pricesData, span=n), index=pricesData.index) return ema
def plot_scatter(data, ax, text, color, span=10, method='emwa'): from pandas.stats.moments import ewma data = ewma(data,span=span) ax.plot(data, color=color, linewidth=1, label=text)
import numpy as np import matplotlib.pyplot as plt import pandas.stats.moments as moments import parsetmy import getwunderground typical = parsetmy.load() current = getwunderground.load() fig = plt.figure() ax1 = fig.add_subplot(111) #ax1.plot(r1.date_mmddyyyy, r1.dewpoint_c, 'o-') ax1.plot(typical.date_mmddyyyy, moments.ewma(typical.dewpoint_c,span=24*14), 'o-') ax1.plot(current['datetime'], moments.ewma(current['dewptm'],span=24), 'o-') #ax2 = fig.add_subplot(212) #ax2.plot(r2.date_mmddyyyy, r2.dewpoint_c, 'o-') fig.autofmt_xdate() plt.show()
except: print EWMA10 raise #V_MA.append(x_mat[i][6]) #arr_concat = np.array([[x_mat[i,c_ind]], [x_mat[i+1,c_ind]]]) #KALMAN.append(arr_concat) #cycle, trend = stats_filters.hpfilter(HP_FILTER, lamb=1600) #sys.exit(0) #X_SHARPE = getSharpe(np.array(X_SHARPE).ravel(), 3) #X_SHARPE2 = getSharpe(np.array(X_SHARPE2).ravel(), 10) SO = stochasticOscillator(np.array(SO), 3) #SO2 = stochasticOscillator(np.array(SO2),10) CHANGE_U = moments.ewma(np.array(CHANGE_U).ravel(), span=3) CHANGE_D = moments.ewma(np.array(CHANGE_D).ravel(), span=3) #TP_SMA = moments.rolling_mean(np.array(TP),window=20,min_periods=0) #KALMAN = np.array(KALMAN) #KALMAN.reshape(len(KALMAN[:,0:1]), len(KALMAN[1:,:])) #CHANGE_U = moments.rolling_mean(np.array(CHANGE_U).ravel(), window=5, min_periods=0) #CHANGE_D = moments.rolling_mean(np.array(CHANGE_D).ravel(), window=5, min_periods=0) #ACC_DIST1 = moments.ewma(np.array(ACC_DIST).ravel(), span=3) #ACC_DIST2 = moments.ewma(np.array(ACC_DIST).ravel(), span=5) #print SO[0:10] #print X_SHARPE[20:30] #print X_SHARPE2[20:30] #print len(X_MA) #print len(X_MA2)
except: print EWMA10 raise #V_MA.append(x_mat[i][6]) #arr_concat = np.array([[x_mat[i,c_ind]], [x_mat[i+1,c_ind]]]) #KALMAN.append(arr_concat) #cycle, trend = stats_filters.hpfilter(HP_FILTER, lamb=1600) #sys.exit(0) #X_SHARPE = getSharpe(np.array(X_SHARPE).ravel(), 3) #X_SHARPE2 = getSharpe(np.array(X_SHARPE2).ravel(), 10) SO = stochasticOscillator(np.array(SO),3) #SO2 = stochasticOscillator(np.array(SO2),10) CHANGE_U = moments.ewma(np.array(CHANGE_U).ravel(), span=3) CHANGE_D = moments.ewma(np.array(CHANGE_D).ravel(), span=3) #TP_SMA = moments.rolling_mean(np.array(TP),window=20,min_periods=0) #KALMAN = np.array(KALMAN) #KALMAN.reshape(len(KALMAN[:,0:1]), len(KALMAN[1:,:])) #CHANGE_U = moments.rolling_mean(np.array(CHANGE_U).ravel(), window=5, min_periods=0) #CHANGE_D = moments.rolling_mean(np.array(CHANGE_D).ravel(), window=5, min_periods=0) #ACC_DIST1 = moments.ewma(np.array(ACC_DIST).ravel(), span=3) #ACC_DIST2 = moments.ewma(np.array(ACC_DIST).ravel(), span=5) #print SO[0:10] #print X_SHARPE[20:30] #print X_SHARPE2[20:30] #print len(X_MA)