def test_streaming(): a = np.array([1, 1, 2, 3, 5, 8, 13], dtype=float) r = stream.MOM(a, timeperiod=1) assert_equals(r, 5) r = stream.MOM(a, timeperiod=2) assert_equals(r, 8) r = stream.MOM(a, timeperiod=3) assert_equals(r, 10) r = stream.MOM(a, timeperiod=4) assert_equals(r, 11) r = stream.MOM(a, timeperiod=5) assert_equals(r, 12) r = stream.MOM(a, timeperiod=6) assert_equals(r, 12) r = stream.MOM(a, timeperiod=7) assert_true(np.isnan(r))
def test_streaming_pandas(): a = pd.Series([1, 1, 2, 3, 5, 8, 13]) r = stream.MOM(a, timeperiod=1) assert_equals(r, 5) r = stream.MOM(a, timeperiod=2) assert_equals(r, 8) r = stream.MOM(a, timeperiod=3) assert_equals(r, 10) r = stream.MOM(a, timeperiod=4) assert_equals(r, 11) r = stream.MOM(a, timeperiod=5) assert_equals(r, 12) r = stream.MOM(a, timeperiod=6) assert_equals(r, 12) r = stream.MOM(a, timeperiod=7) assert_true(np.isnan(r))
def test_streaming(): a = np.array([1,1,2,3,5,8,13], dtype=float) r = stream.MOM(a, timeperiod=1) assert r == 5 r = stream.MOM(a, timeperiod=2) assert r == 8 r = stream.MOM(a, timeperiod=3) assert r == 10 r = stream.MOM(a, timeperiod=4) assert r == 11 r = stream.MOM(a, timeperiod=5) assert r == 12 r = stream.MOM(a, timeperiod=6) assert r == 12 r = stream.MOM(a, timeperiod=7) assert np.isnan(r)
def test_streaming_pandas(): a = pd.Series([1,1,2,3,5,8,13]) r = stream.MOM(a, timeperiod=1) assert r == 5 r = stream.MOM(a, timeperiod=2) assert r == 8 r = stream.MOM(a, timeperiod=3) assert r == 10 r = stream.MOM(a, timeperiod=4) assert r == 11 r = stream.MOM(a, timeperiod=5) assert r == 12 r = stream.MOM(a, timeperiod=6) assert r == 12 r = stream.MOM(a, timeperiod=7) assert np.isnan(r)
def _build_indicators(self, df): if not self.realtime: inputs = df.to_dict(orient="list") for col in inputs: inputs[col] = np.array(inputs[col]) c = df["close"] for n in range(2, 40): inputs["bband_u_" + str(n)], inputs["bband_m_" + str(n)], inputs["bband_l_" + str(n)] = ta.BBANDS( inputs, n) inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n) inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n) # fast_ema = c.ewm(span = n, adjust = False).mean() # slow_ema = c.ewm(span = n*2, adjust = False).mean() # macd1 = fast_ema - slow_ema # macd2 = macd1.ewm(span = int(n*2/3), adjust = False).mean() # macd3 = macd1 - macd2 # inputs["macd_"+str(n)] = macd1.values # inputs["macdsignal_"+str(n)] = macd2.values # inputs["macdhist_"+str(n)] = macd3.values if n != 2: inputs["macd_" + str(n)], inputs["macdsignal_" + str(n)], inputs["macdhist_" + str(n)] = ta.MACD( inputs, n, n * 2, int(n * 2 / 3)) else: inputs["macd_" + str(n)], inputs["macdsignal_" + str(n)], inputs["macdhist_" + str(n)] = ta.MACD( inputs, n, n * 2, 1) # macd = [macd1.values, macd2.values, macd3.values] # for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]): # for day in zip(inputs[i], macd[idx]): # print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0])) inputs["mfi_" + str(n)] = ta.MFI(inputs, n) inputs["ult_" + str(n)] = ta.ULTOSC(inputs, n, n * 2, n * 4) inputs["willr_" + str(n)] = ta.WILLR(inputs, n) inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs) inputs["mom_" + str(n)] = ta.MOM(inputs, n) inputs["volume"] = list(map(lambda x: x / 10000, inputs["volume"])) df = pd.DataFrame().from_dict(inputs) # df = df.ix[100:] # print(df.tail(5)["macd_3"], df.tail(5)["macdsignal_3"], df.tail(5)["macdhist_3"]) return df else: # Build data one-by-one, as if it's coming in one at a time output = pd.DataFrame() sliding_window = pd.DataFrame() for idx, day in df.iterrows(): print("\rNow building day", str(idx), end="", flush=True) day = copy.deepcopy(day) # Avoid reference vs copy bullshit sliding_window = sliding_window.append(day, ignore_index=True) # print(day, type(day)) day_out = {} # print(sliding_window) o = sliding_window["open"].values h = sliding_window["high"].values l = sliding_window["low"].values c_series = sliding_window["close"] c = sliding_window["close"].values # print("----") # print(c) v = sliding_window["volume"].values for t in ["open", "high", "low", "close"]: day_out[t] = sliding_window[t].values[-1] for n in range(2, 40): # time.sleep(0.1) day_out["bband_u_" + str(n)], day_out["bband_m_" + str(n)], day_out[ "bband_l_" + str(n)] = stream.BBANDS(c, n) day_out["sma_" + str(n)] = stream.SMA(c, timeperiod=n) day_out["adx_" + str(n)] = stream.ADX(h, l, c, timeperiod=n) fast_ema = c_series.ewm(span=n, adjust=False).mean() slow_ema = c_series.ewm(span=n * 2, adjust=False).mean() macd1 = fast_ema - slow_ema macd2 = macd1.ewm(span=int(n * 2 / 3), adjust=False).mean() macd3 = macd1 - macd2 day_out["macd_" + str(n)] = macd1.values[-1] day_out["macdsignal_" + str(n)] = macd2.values[-1] day_out["macdhist_" + str(n)] = macd3.values[-1] # if n != 2: # day_out["macd_"+str(n)], day_out["macdsignal_"+str(n)], day_out["macdhist_"+str(n)] = stream.MACD(c, n, n*2, int(n*2/3)) # elif idx > 100: # macd = ta.MACD({"close":c}, n, n*2, 1) # day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = (x[-1] for x in macd) # else: # day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = None, None, None # macd = [macd1.values, macd2.values, macd3.values] # for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]): # for day in zip(inputs[i], macd[idx]): # print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0])) day_out["mfi_" + str(n)] = stream.MFI(h, l, c, v, n) day_out["ult_" + str(n)] = stream.ULTOSC( h, l, c, n, n * 2, n * 4) day_out["willr_" + str(n)] = stream.WILLR(h, l, c, n) day_out["slowk"], day_out["slowd"] = stream.STOCH(h, l, c) day_out["mom_" + str(n)] = stream.MOM(c, n) day_out["volume"] = v[-1] / 10000 # print(day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"]) output = output.append(day_out, ignore_index=True) # print(output.tail(5)["macd_3"], output.tail(5)["macdsignal_3"], output.tail(5)["macdhist_3"]) return output
def dataproc_worker(start, end, live, ns, data=None): while True: # print(start,end, "gonna call the DB and get:") start = start.replace(microsecond=0) end = end.replace(microsecond=0) flip = False # if not live: # if randrange(0, 10) > 4: flip = True # else: flip = False if data is None: globalvar.dbcur.execute( "SELECT * FROM futures WHERE time BETWEEN (?) AND (?)", [(start - datetime(1970, 1, 1)) / timedelta(seconds=1), (end - datetime(1970, 1, 1)) / timedelta(seconds=1)]) data = globalvar.dbcur.fetchall() # benchstart = datetime.utcnow() data = [list(elem) for elem in data] # print("#1:", data[0], data[-1]) # for index, line in enumerate(data): # # data[index][0] = datetime.strptime(line[0], "%Y-%m-%dT%H:%M:%SZ") #problem 2 # date = line[0] # string of '2016-07-07T10:36:42Z' using explicit slicing: # data[index][0] = datetime(int(date[:4]), int(date[5:7]), int(date[8:10]), int(date[11:13]), # int(date[14:16]), int(date[17:19])) # averageNumerator = 0.0 # print("#2:",data[0],data[-1]) datalength = len(data) if not live: finaltime = len(data) - 120 # print(data[-1][0], "as finaltime", data[-1][1], "as lastlast") # consistently accurate for index, line in enumerate(data): if live: averageNumerator += line[4] elif index < finaltime: averageNumerator += line[4] # if (data[-1][0] == data[-120][0]): break # here is where we do live adjustments remove 6k seconds at start pad 6k 0's at end # print("#3", data[0], data[-1], len(data)) averagePrice = averageNumerator / (len(data) - 120) if live: del data[0:121] for index, line in enumerate( data): # normalize to the 120 mins average price data[index][1] = line[1] / averagePrice # o data[index][2] = line[2] / averagePrice # h data[index][3] = line[3] / averagePrice # l data[index][4] = line[4] / averagePrice # c timeperiods = [ 10, 100, 1000, 10000, 100000 ] # (len(data) - 6000)] is too long, 100k max in C talib # initlist = [data[0][0]] * 86400 # # print (timeperiods) if not live: open = np.array([row[1] for row in data[0:-120]], dtype=np.double) high = np.array([row[2] for row in data[0:-120]], dtype=np.double) low = np.array([row[3] for row in data[0:-120]], dtype=np.double) close = np.array([row[4] for row in data[0:-120]], dtype=np.double) volume = np.array([row[5] for row in data[0:-120]], dtype=np.double) else: open = np.array([row[1] for row in data[0:-1]], dtype=np.double) high = np.array([row[2] for row in data[0:-1]], dtype=np.double) low = np.array([row[3] for row in data[0:-1]], dtype=np.double) close = np.array([row[4] for row in data[0:-1]], dtype=np.double) volume = np.array([row[5] for row in data[0:-1]], dtype=np.double) # print("internal last:", close[-1]) if flip: open = (open * -1) + 2 high = (high * -1) + 2 low = (low * -1) + 2 close = (close * -1) + 2 data = np.array([row[4] for row in data[0:-1]], dtype=np.double) data = (data * -1) + 2 # let's do the non-timeperiod indicators first # print("problem four took " + str(datetime.utcnow() - benchstart)) # print("starting TA") trange = ta.TRANGE(high, low, close) obv = ta.OBV(close, volume) ad = ta.AD(high, low, close, volume) ht_trendline = ta.HT_TRENDLINE(close) ht_dcperiod = ta.HT_DCPERIOD(close) ht_dcphase = ta.HT_DCPHASE(close) ht_phasor_inphase, ht_phasor_quadrature = ta.HT_PHASOR(close) ht_sine, ht_leadsine = ta.HT_SINE(close) ht_trendmode = ta.HT_TRENDMODE(close) mama, fama = ta.MAMA(close, fastlimit=.5, slowlimit=.05) sar = ta.SAR(high, low) # could crash, no extended params supplied sarext = ta.SAREXT(high, low) # this one too bop = ta.BOP(open, high, low, close) # print("basic TA took " + str(datetime.utcnow() - benchstart)) # print("starting timeperiod TA") # setting up timeperiod-requiring indicators slowk, slowd, fastk, fastd, rsik, rsid, t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci, cmo, dx, macd, \ macdsignal, macdhist, macdext, macdsignalext, macdhistext, macdfix, macdhistfix, macdsignalfix, mfi, minus_dm, \ minus_di, mom, plus_di, plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr, beta, correl, linearreg, \ linearreg_angle, linearreg_intercept, linearreg_slope, stddev, tsf, var, atr, natr, adosc, upperband, \ middleband, lowerband, dema, ema, kama, ma, midpoint, midprice, sma, tema, trima, wma = ([] for i in range( 63)) # counted 63 twice for timeperiod in timeperiods: benchstart = datetime.utcnow() fastperiod = int(timeperiod / 2) slowperiod = timeperiod stochk = int(timeperiod / 3) stochd = int(timeperiod / 4) # n = int(len(close) / slicefactor) # offset = (len(close)-1) - (n * slicefactor) # opens= open[offset::slicefactor] # high= high[offset::slicefactor] # low= low[offset::slicefactor] # close= close[offset::slicefactor] # volumes= volume[offset::slicefactor] # now for the weird indicators matype=1 for life, ema stoch1_temp, stoch2_temp = ta.STOCH(high, low, close, fastk_period=stochk, slowk_period=stochd, slowk_matype=1, slowd_period=stochd, slowd_matype=1) slowk.append(stoch1_temp) slowd.append(stoch2_temp) stoch1_temp, stoch2_temp = ta.STOCHF(high, low, close, fastk_period=stochk, fastd_period=stochd, fastd_matype=1) fastk.append(stoch1_temp) fastd.append(stoch2_temp) stoch1_temp, stoch2_temp = ta.STOCHRSI(close, timeperiod=timeperiod, fastk_period=stochk, fastd_period=stochd, fastd_matype=1) rsik.append(stoch1_temp) rsid.append(stoch2_temp) t3.append(ta.T3(close, timeperiod=timeperiod, vfactor=.7)) # now for the normal indicators # momentum adx.append(ta.ADX(high, low, close, timeperiod=timeperiod)) adxr.append(ta.ADXR(high, low, close, timeperiod=timeperiod)) apo.append( ta.APO(close, fastperiod=fastperiod, slowperiod=slowperiod)) aroondown_temp, aroonup_temp = ta.AROON(high, low, timeperiod=timeperiod) aroondown.append(aroondown_temp) aroonup.append(aroonup_temp) aroonosc.append(ta.AROONOSC(high, low, timeperiod=timeperiod)) cci.append(ta.CCI(high, low, close, timeperiod=timeperiod)) cmo.append(ta.CMO(close, timeperiod=timeperiod)) dx.append(ta.DX(high, low, close, timeperiod=timeperiod)) macd_temp, macdsignal_temp, macdhist_temp = ta.MACD( close, fastperiod=fastperiod, slowperiod=slowperiod, signalperiod=stochk) macd.append(macd_temp) macdsignal.append(macdsignal_temp) macdhist.append(macdhist_temp) macd_temp, macdsignal_temp, macdhist_temp = ta.MACDEXT( close, fastperiod=fastperiod, fastmatype=1, slowperiod=slowperiod, slowmatype=1, signalperiod=stochk, signalmatype=1) macdext.append(macd_temp) macdsignalext.append(macdsignal_temp) macdhistext.append(macdhist_temp) macd_temp, macdsignal_temp, macdhist_temp = ta.MACDFIX( close, signalperiod=stochk) macdfix.append(macd_temp) macdsignalfix.append(macdsignal_temp) macdhistfix.append(macdhist_temp) mfi.append(ta.MFI(high, low, close, volume, timeperiod=timeperiod)) minus_di.append( ta.MINUS_DI(high, low, close, timeperiod=timeperiod)) minus_dm.append(ta.MINUS_DM(high, low, timeperiod=timeperiod)) mom.append(ta.MOM(close, timeperiod=timeperiod)) plus_di.append(ta.PLUS_DI(high, low, close, timeperiod=timeperiod)) plus_dm.append(ta.PLUS_DM(high, low, timeperiod=timeperiod)) ppo.append( ta.PPO(close, fastperiod=fastperiod, slowperiod=slowperiod, matype=1)) roc.append(ta.ROC(close, timeperiod=timeperiod)) rocr.append(ta.ROCR(close, timeperiod=timeperiod)) rsi.append(ta.RSI(close, timeperiod=timeperiod)) trix.append(ta.TRIX(close, timeperiod=timeperiod)) ultosc.append( ta.ULTOSC(high, low, close, timeperiod1=stochk, timeperiod2=fastperiod, timeperiod3=timeperiod)) willr.append(ta.WILLR(high, low, close, timeperiod=timeperiod)) # end momentum, begin statistics beta.append(ta.BETA(high, low, timeperiod=timeperiod)) correl.append(ta.CORREL(high, low, timeperiod=timeperiod)) linearreg.append(ta.LINEARREG(close, timeperiod=timeperiod)) linearreg_angle.append( ta.LINEARREG_ANGLE(close, timeperiod=timeperiod)) linearreg_intercept.append( ta.LINEARREG_INTERCEPT(close, timeperiod=timeperiod)) linearreg_slope.append( ta.LINEARREG_SLOPE(close, timeperiod=timeperiod)) stddev.append(ta.STDDEV(close, timeperiod=timeperiod, nbdev=1)) tsf.append(ta.TSF(close, timeperiod=timeperiod)) var.append(ta.VAR(close, timeperiod=timeperiod, nbdev=1)) # end stats, begin vol atr.append(ta.ATR(high, low, close, timeperiod=timeperiod)) natr.append(ta.NATR(high, low, close, timeperiod=timeperiod)) adosc.append( ta.ADOSC(high, low, close, volume, fastperiod=fastperiod, slowperiod=slowperiod)) # end vol, begin overlaps upperband_temp, middleband_temp, lowerband_temp = ta.BBANDS( close, timeperiod=timeperiod, matype=1) upperband.append(upperband_temp) middleband.append(middleband_temp) lowerband.append(lowerband_temp) dema.append(ta.DEMA(close, timeperiod=timeperiod)) ema.append(ta.EMA(close, timeperiod=timeperiod)) kama.append(ta.KAMA(close, timeperiod=timeperiod)) ma.append(ta.MA(close, timeperiod=timeperiod, matype=0)) midpoint.append(ta.MIDPOINT(close, timeperiod=timeperiod)) midprice.append(ta.MIDPRICE(high, low, timeperiod=timeperiod)) sma.append(ta.SMA(close, timeperiod=timeperiod)) tema.append(ta.TEMA(close, timeperiod=timeperiod)) trima.append(ta.TRIMA(close, timeperiod=timeperiod)) wma.append(ta.WMA(close, timeperiod=timeperiod)) # print("finishing timeperiod" + str(timeperiod)) # print("TA for " + str(timeperiod) + " took " + str(datetime.utcnow() - benchstart)) # 1 min, 10min, 100m, currentprice, indicators if live: finishedline = (close[-1]) else: if not flip: finishedline = ( # date, flip, 1, 10, 20, 40, 80, 120, current price (end - datetime(1970, 1, 1)) / timedelta(seconds=1), 0, data[-120][4], data[-111][4], data[-101][4], data[-81][4], data[-41][4], data[-1][4], data[-121][4]) if flip: finishedline = ( # date, flip, 1, 10, 20, 40, 80, 120, current price (end - datetime(1970, 1, 1)) / timedelta(seconds=1), 1, data[-120], data[-111], data[-101], data[-81], data[-41], data[-1], data[-121]) finishedline += (trange, obv, ad, ht_trendline, ht_dcperiod, ht_dcphase, ht_phasor_inphase, ht_phasor_quadrature, ht_sine, ht_leadsine, ht_trendmode, mama, fama, sar, sarext, bop, slowk, slowd, fastk, fastd, rsik, rsid, t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci, cmo, dx, macd, macdsignal, macdhist, macdext, macdsignalext, macdhistext, macdfix, macdsignalfix, macdhistfix, mfi, minus_di, minus_dm, mom, plus_di, plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr, beta, correl, linearreg, linearreg_angle, linearreg_intercept, linearreg_slope, stddev, tsf, var, atr, natr, adosc, upperband, middleband, lowerband, dema, ema, kama, ma, midpoint, midprice, sma, tema, trima, wma) finishedline = flatten(finishedline) if live: finishedline = np.array(finishedline, dtype=np.double) finishedline = finishedline[~np.isnan(finishedline)] # remove nans else: finishedline = np.array(finishedline, dtype=np.double) finishedline = finishedline[~np.isnan(finishedline)] # print(pid + " has finished") if len(finishedline) != 326: break return finishedline break