def test_streaming():
a = np.array([1, 1, 2, 3, 5, 8, 13], dtype=float)
r = stream.MOM(a, timeperiod=1)
assert_equals(r, 5)
r = stream.MOM(a, timeperiod=2)
assert_equals(r, 8)
r = stream.MOM(a, timeperiod=3)
assert_equals(r, 10)
r = stream.MOM(a, timeperiod=4)
assert_equals(r, 11)
r = stream.MOM(a, timeperiod=5)
assert_equals(r, 12)
r = stream.MOM(a, timeperiod=6)
assert_equals(r, 12)
r = stream.MOM(a, timeperiod=7)
assert_true(np.isnan(r))
def test_streaming_pandas():
a = pd.Series([1, 1, 2, 3, 5, 8, 13])
r = stream.MOM(a, timeperiod=1)
assert_equals(r, 5)
r = stream.MOM(a, timeperiod=2)
assert_equals(r, 8)
r = stream.MOM(a, timeperiod=3)
assert_equals(r, 10)
r = stream.MOM(a, timeperiod=4)
assert_equals(r, 11)
r = stream.MOM(a, timeperiod=5)
assert_equals(r, 12)
r = stream.MOM(a, timeperiod=6)
assert_equals(r, 12)
r = stream.MOM(a, timeperiod=7)
assert_true(np.isnan(r))
def test_streaming():
a = np.array([1,1,2,3,5,8,13], dtype=float)
r = stream.MOM(a, timeperiod=1)
assert r == 5
r = stream.MOM(a, timeperiod=2)
assert r == 8
r = stream.MOM(a, timeperiod=3)
assert r == 10
r = stream.MOM(a, timeperiod=4)
assert r == 11
r = stream.MOM(a, timeperiod=5)
assert r == 12
r = stream.MOM(a, timeperiod=6)
assert r == 12
r = stream.MOM(a, timeperiod=7)
assert np.isnan(r)
def test_streaming_pandas():
a = pd.Series([1,1,2,3,5,8,13])
r = stream.MOM(a, timeperiod=1)
assert r == 5
r = stream.MOM(a, timeperiod=2)
assert r == 8
r = stream.MOM(a, timeperiod=3)
assert r == 10
r = stream.MOM(a, timeperiod=4)
assert r == 11
r = stream.MOM(a, timeperiod=5)
assert r == 12
r = stream.MOM(a, timeperiod=6)
assert r == 12
r = stream.MOM(a, timeperiod=7)
assert np.isnan(r)
def _build_indicators(self, df):
if not self.realtime:
inputs = df.to_dict(orient="list")
for col in inputs:
inputs[col] = np.array(inputs[col])
c = df["close"]
for n in range(2, 40):
inputs["bband_u_" +
str(n)], inputs["bband_m_" +
str(n)], inputs["bband_l_" +
str(n)] = ta.BBANDS(
inputs, n)
inputs["sma_" + str(n)] = ta.SMA(inputs, timeperiod=n)
inputs["adx_" + str(n)] = ta.ADX(inputs, timeperiod=n)
# fast_ema = c.ewm(span = n, adjust = False).mean()
# slow_ema = c.ewm(span = n*2, adjust = False).mean()
# macd1 = fast_ema - slow_ema
# macd2 = macd1.ewm(span = int(n*2/3), adjust = False).mean()
# macd3 = macd1 - macd2
# inputs["macd_"+str(n)] = macd1.values
# inputs["macdsignal_"+str(n)] = macd2.values
# inputs["macdhist_"+str(n)] = macd3.values
if n != 2:
inputs["macd_" +
str(n)], inputs["macdsignal_" +
str(n)], inputs["macdhist_" +
str(n)] = ta.MACD(
inputs, n,
n * 2,
int(n * 2 / 3))
else:
inputs["macd_" +
str(n)], inputs["macdsignal_" +
str(n)], inputs["macdhist_" +
str(n)] = ta.MACD(
inputs, n,
n * 2, 1)
# macd = [macd1.values, macd2.values, macd3.values]
# for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]):
# for day in zip(inputs[i], macd[idx]):
# print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0]))
inputs["mfi_" + str(n)] = ta.MFI(inputs, n)
inputs["ult_" + str(n)] = ta.ULTOSC(inputs, n, n * 2, n * 4)
inputs["willr_" + str(n)] = ta.WILLR(inputs, n)
inputs["slowk"], inputs["slowd"] = ta.STOCH(inputs)
inputs["mom_" + str(n)] = ta.MOM(inputs, n)
inputs["volume"] = list(map(lambda x: x / 10000, inputs["volume"]))
df = pd.DataFrame().from_dict(inputs)
# df = df.ix[100:]
# print(df.tail(5)["macd_3"], df.tail(5)["macdsignal_3"], df.tail(5)["macdhist_3"])
return df
else:
# Build data one-by-one, as if it's coming in one at a time
output = pd.DataFrame()
sliding_window = pd.DataFrame()
for idx, day in df.iterrows():
print("\rNow building day", str(idx), end="", flush=True)
day = copy.deepcopy(day) # Avoid reference vs copy bullshit
sliding_window = sliding_window.append(day, ignore_index=True)
# print(day, type(day))
day_out = {}
# print(sliding_window)
o = sliding_window["open"].values
h = sliding_window["high"].values
l = sliding_window["low"].values
c_series = sliding_window["close"]
c = sliding_window["close"].values
# print("----")
# print(c)
v = sliding_window["volume"].values
for t in ["open", "high", "low", "close"]:
day_out[t] = sliding_window[t].values[-1]
for n in range(2, 40):
# time.sleep(0.1)
day_out["bband_u_" +
str(n)], day_out["bband_m_" + str(n)], day_out[
"bband_l_" + str(n)] = stream.BBANDS(c, n)
day_out["sma_" + str(n)] = stream.SMA(c, timeperiod=n)
day_out["adx_" + str(n)] = stream.ADX(h,
l,
c,
timeperiod=n)
fast_ema = c_series.ewm(span=n, adjust=False).mean()
slow_ema = c_series.ewm(span=n * 2, adjust=False).mean()
macd1 = fast_ema - slow_ema
macd2 = macd1.ewm(span=int(n * 2 / 3), adjust=False).mean()
macd3 = macd1 - macd2
day_out["macd_" + str(n)] = macd1.values[-1]
day_out["macdsignal_" + str(n)] = macd2.values[-1]
day_out["macdhist_" + str(n)] = macd3.values[-1]
# if n != 2:
# day_out["macd_"+str(n)], day_out["macdsignal_"+str(n)], day_out["macdhist_"+str(n)] = stream.MACD(c, n, n*2, int(n*2/3))
# elif idx > 100:
# macd = ta.MACD({"close":c}, n, n*2, 1)
# day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = (x[-1] for x in macd)
# else:
# day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"] = None, None, None
# macd = [macd1.values, macd2.values, macd3.values]
# for idx, i in enumerate(["macd_"+str(n), "macdsignal_"+str(n), "macdhist_"+str(n)]):
# for day in zip(inputs[i], macd[idx]):
# print("Type: %s N: %d PD: %.3f TA: %.3f, " % (i, n, day[1], day[0]))
day_out["mfi_" + str(n)] = stream.MFI(h, l, c, v, n)
day_out["ult_" + str(n)] = stream.ULTOSC(
h, l, c, n, n * 2, n * 4)
day_out["willr_" + str(n)] = stream.WILLR(h, l, c, n)
day_out["slowk"], day_out["slowd"] = stream.STOCH(h, l, c)
day_out["mom_" + str(n)] = stream.MOM(c, n)
day_out["volume"] = v[-1] / 10000
# print(day_out["macd_2"], day_out["macdsignal_2"], day_out["macdhist_2"])
output = output.append(day_out, ignore_index=True)
# print(output.tail(5)["macd_3"], output.tail(5)["macdsignal_3"], output.tail(5)["macdhist_3"])
return output
def dataproc_worker(start, end, live, ns, data=None):
while True:
# print(start,end, "gonna call the DB and get:")
start = start.replace(microsecond=0)
end = end.replace(microsecond=0)
flip = False
# if not live:
# if randrange(0, 10) > 4: flip = True
# else: flip = False
if data is None:
globalvar.dbcur.execute(
"SELECT * FROM futures WHERE time BETWEEN (?) AND (?)",
[(start - datetime(1970, 1, 1)) / timedelta(seconds=1),
(end - datetime(1970, 1, 1)) / timedelta(seconds=1)])
data = globalvar.dbcur.fetchall()
# benchstart = datetime.utcnow()
data = [list(elem) for elem in data]
# print("#1:", data[0], data[-1])
# for index, line in enumerate(data):
# # data[index][0] = datetime.strptime(line[0], "%Y-%m-%dT%H:%M:%SZ") #problem 2
# date = line[0] # string of '2016-07-07T10:36:42Z' using explicit slicing:
# data[index][0] = datetime(int(date[:4]), int(date[5:7]), int(date[8:10]), int(date[11:13]),
# int(date[14:16]), int(date[17:19]))
#
averageNumerator = 0.0
# print("#2:",data[0],data[-1])
datalength = len(data)
if not live:
finaltime = len(data) - 120
# print(data[-1][0], "as finaltime", data[-1][1], "as lastlast") # consistently accurate
for index, line in enumerate(data):
if live:
averageNumerator += line[4]
elif index < finaltime:
averageNumerator += line[4]
# if (data[-1][0] == data[-120][0]): break
# here is where we do live adjustments remove 6k seconds at start pad 6k 0's at end
# print("#3", data[0], data[-1], len(data))
averagePrice = averageNumerator / (len(data) - 120)
if live:
del data[0:121]
for index, line in enumerate(
data): # normalize to the 120 mins average price
data[index][1] = line[1] / averagePrice # o
data[index][2] = line[2] / averagePrice # h
data[index][3] = line[3] / averagePrice # l
data[index][4] = line[4] / averagePrice # c
timeperiods = [
10, 100, 1000, 10000, 100000
] # (len(data) - 6000)] is too long, 100k max in C talib
# initlist = [data[0][0]] * 86400
# # print (timeperiods)
if not live:
open = np.array([row[1] for row in data[0:-120]], dtype=np.double)
high = np.array([row[2] for row in data[0:-120]], dtype=np.double)
low = np.array([row[3] for row in data[0:-120]], dtype=np.double)
close = np.array([row[4] for row in data[0:-120]], dtype=np.double)
volume = np.array([row[5] for row in data[0:-120]],
dtype=np.double)
else:
open = np.array([row[1] for row in data[0:-1]], dtype=np.double)
high = np.array([row[2] for row in data[0:-1]], dtype=np.double)
low = np.array([row[3] for row in data[0:-1]], dtype=np.double)
close = np.array([row[4] for row in data[0:-1]], dtype=np.double)
volume = np.array([row[5] for row in data[0:-1]], dtype=np.double)
# print("internal last:", close[-1])
if flip:
open = (open * -1) + 2
high = (high * -1) + 2
low = (low * -1) + 2
close = (close * -1) + 2
data = np.array([row[4] for row in data[0:-1]], dtype=np.double)
data = (data * -1) + 2
# let's do the non-timeperiod indicators first
# print("problem four took " + str(datetime.utcnow() - benchstart))
# print("starting TA")
trange = ta.TRANGE(high, low, close)
obv = ta.OBV(close, volume)
ad = ta.AD(high, low, close, volume)
ht_trendline = ta.HT_TRENDLINE(close)
ht_dcperiod = ta.HT_DCPERIOD(close)
ht_dcphase = ta.HT_DCPHASE(close)
ht_phasor_inphase, ht_phasor_quadrature = ta.HT_PHASOR(close)
ht_sine, ht_leadsine = ta.HT_SINE(close)
ht_trendmode = ta.HT_TRENDMODE(close)
mama, fama = ta.MAMA(close, fastlimit=.5, slowlimit=.05)
sar = ta.SAR(high, low) # could crash, no extended params supplied
sarext = ta.SAREXT(high, low) # this one too
bop = ta.BOP(open, high, low, close)
# print("basic TA took " + str(datetime.utcnow() - benchstart))
# print("starting timeperiod TA")
# setting up timeperiod-requiring indicators
slowk, slowd, fastk, fastd, rsik, rsid, t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci, cmo, dx, macd, \
macdsignal, macdhist, macdext, macdsignalext, macdhistext, macdfix, macdhistfix, macdsignalfix, mfi, minus_dm, \
minus_di, mom, plus_di, plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr, beta, correl, linearreg, \
linearreg_angle, linearreg_intercept, linearreg_slope, stddev, tsf, var, atr, natr, adosc, upperband, \
middleband, lowerband, dema, ema, kama, ma, midpoint, midprice, sma, tema, trima, wma = ([] for i in range(
63)) # counted 63 twice
for timeperiod in timeperiods:
benchstart = datetime.utcnow()
fastperiod = int(timeperiod / 2)
slowperiod = timeperiod
stochk = int(timeperiod / 3)
stochd = int(timeperiod / 4)
# n = int(len(close) / slicefactor)
# offset = (len(close)-1) - (n * slicefactor)
# opens= open[offset::slicefactor]
# high= high[offset::slicefactor]
# low= low[offset::slicefactor]
# close= close[offset::slicefactor]
# volumes= volume[offset::slicefactor]
# now for the weird indicators matype=1 for life, ema
stoch1_temp, stoch2_temp = ta.STOCH(high,
low,
close,
fastk_period=stochk,
slowk_period=stochd,
slowk_matype=1,
slowd_period=stochd,
slowd_matype=1)
slowk.append(stoch1_temp)
slowd.append(stoch2_temp)
stoch1_temp, stoch2_temp = ta.STOCHF(high,
low,
close,
fastk_period=stochk,
fastd_period=stochd,
fastd_matype=1)
fastk.append(stoch1_temp)
fastd.append(stoch2_temp)
stoch1_temp, stoch2_temp = ta.STOCHRSI(close,
timeperiod=timeperiod,
fastk_period=stochk,
fastd_period=stochd,
fastd_matype=1)
rsik.append(stoch1_temp)
rsid.append(stoch2_temp)
t3.append(ta.T3(close, timeperiod=timeperiod, vfactor=.7))
# now for the normal indicators
# momentum
adx.append(ta.ADX(high, low, close, timeperiod=timeperiod))
adxr.append(ta.ADXR(high, low, close, timeperiod=timeperiod))
apo.append(
ta.APO(close, fastperiod=fastperiod, slowperiod=slowperiod))
aroondown_temp, aroonup_temp = ta.AROON(high,
low,
timeperiod=timeperiod)
aroondown.append(aroondown_temp)
aroonup.append(aroonup_temp)
aroonosc.append(ta.AROONOSC(high, low, timeperiod=timeperiod))
cci.append(ta.CCI(high, low, close, timeperiod=timeperiod))
cmo.append(ta.CMO(close, timeperiod=timeperiod))
dx.append(ta.DX(high, low, close, timeperiod=timeperiod))
macd_temp, macdsignal_temp, macdhist_temp = ta.MACD(
close,
fastperiod=fastperiod,
slowperiod=slowperiod,
signalperiod=stochk)
macd.append(macd_temp)
macdsignal.append(macdsignal_temp)
macdhist.append(macdhist_temp)
macd_temp, macdsignal_temp, macdhist_temp = ta.MACDEXT(
close,
fastperiod=fastperiod,
fastmatype=1,
slowperiod=slowperiod,
slowmatype=1,
signalperiod=stochk,
signalmatype=1)
macdext.append(macd_temp)
macdsignalext.append(macdsignal_temp)
macdhistext.append(macdhist_temp)
macd_temp, macdsignal_temp, macdhist_temp = ta.MACDFIX(
close, signalperiod=stochk)
macdfix.append(macd_temp)
macdsignalfix.append(macdsignal_temp)
macdhistfix.append(macdhist_temp)
mfi.append(ta.MFI(high, low, close, volume, timeperiod=timeperiod))
minus_di.append(
ta.MINUS_DI(high, low, close, timeperiod=timeperiod))
minus_dm.append(ta.MINUS_DM(high, low, timeperiod=timeperiod))
mom.append(ta.MOM(close, timeperiod=timeperiod))
plus_di.append(ta.PLUS_DI(high, low, close, timeperiod=timeperiod))
plus_dm.append(ta.PLUS_DM(high, low, timeperiod=timeperiod))
ppo.append(
ta.PPO(close,
fastperiod=fastperiod,
slowperiod=slowperiod,
matype=1))
roc.append(ta.ROC(close, timeperiod=timeperiod))
rocr.append(ta.ROCR(close, timeperiod=timeperiod))
rsi.append(ta.RSI(close, timeperiod=timeperiod))
trix.append(ta.TRIX(close, timeperiod=timeperiod))
ultosc.append(
ta.ULTOSC(high,
low,
close,
timeperiod1=stochk,
timeperiod2=fastperiod,
timeperiod3=timeperiod))
willr.append(ta.WILLR(high, low, close, timeperiod=timeperiod))
# end momentum, begin statistics
beta.append(ta.BETA(high, low, timeperiod=timeperiod))
correl.append(ta.CORREL(high, low, timeperiod=timeperiod))
linearreg.append(ta.LINEARREG(close, timeperiod=timeperiod))
linearreg_angle.append(
ta.LINEARREG_ANGLE(close, timeperiod=timeperiod))
linearreg_intercept.append(
ta.LINEARREG_INTERCEPT(close, timeperiod=timeperiod))
linearreg_slope.append(
ta.LINEARREG_SLOPE(close, timeperiod=timeperiod))
stddev.append(ta.STDDEV(close, timeperiod=timeperiod, nbdev=1))
tsf.append(ta.TSF(close, timeperiod=timeperiod))
var.append(ta.VAR(close, timeperiod=timeperiod, nbdev=1))
# end stats, begin vol
atr.append(ta.ATR(high, low, close, timeperiod=timeperiod))
natr.append(ta.NATR(high, low, close, timeperiod=timeperiod))
adosc.append(
ta.ADOSC(high,
low,
close,
volume,
fastperiod=fastperiod,
slowperiod=slowperiod))
# end vol, begin overlaps
upperband_temp, middleband_temp, lowerband_temp = ta.BBANDS(
close, timeperiod=timeperiod, matype=1)
upperband.append(upperband_temp)
middleband.append(middleband_temp)
lowerband.append(lowerband_temp)
dema.append(ta.DEMA(close, timeperiod=timeperiod))
ema.append(ta.EMA(close, timeperiod=timeperiod))
kama.append(ta.KAMA(close, timeperiod=timeperiod))
ma.append(ta.MA(close, timeperiod=timeperiod, matype=0))
midpoint.append(ta.MIDPOINT(close, timeperiod=timeperiod))
midprice.append(ta.MIDPRICE(high, low, timeperiod=timeperiod))
sma.append(ta.SMA(close, timeperiod=timeperiod))
tema.append(ta.TEMA(close, timeperiod=timeperiod))
trima.append(ta.TRIMA(close, timeperiod=timeperiod))
wma.append(ta.WMA(close, timeperiod=timeperiod))
# print("finishing timeperiod" + str(timeperiod))
# print("TA for " + str(timeperiod) + " took " + str(datetime.utcnow() - benchstart))
# 1 min, 10min, 100m, currentprice, indicators
if live:
finishedline = (close[-1])
else:
if not flip:
finishedline = ( # date, flip, 1, 10, 20, 40, 80, 120, current price
(end - datetime(1970, 1, 1)) / timedelta(seconds=1), 0,
data[-120][4], data[-111][4], data[-101][4], data[-81][4],
data[-41][4], data[-1][4], data[-121][4])
if flip:
finishedline = ( # date, flip, 1, 10, 20, 40, 80, 120, current price
(end - datetime(1970, 1, 1)) / timedelta(seconds=1), 1,
data[-120], data[-111], data[-101], data[-81], data[-41],
data[-1], data[-121])
finishedline += (trange, obv, ad, ht_trendline, ht_dcperiod,
ht_dcphase, ht_phasor_inphase, ht_phasor_quadrature,
ht_sine, ht_leadsine, ht_trendmode, mama, fama, sar,
sarext, bop, slowk, slowd, fastk, fastd, rsik, rsid,
t3, adx, adxr, apo, aroondown, aroonup, aroonosc, cci,
cmo, dx, macd, macdsignal, macdhist, macdext,
macdsignalext, macdhistext, macdfix, macdsignalfix,
macdhistfix, mfi, minus_di, minus_dm, mom, plus_di,
plus_dm, ppo, roc, rocr, rsi, trix, ultosc, willr,
beta, correl, linearreg, linearreg_angle,
linearreg_intercept, linearreg_slope, stddev, tsf,
var, atr, natr, adosc, upperband, middleband,
lowerband, dema, ema, kama, ma, midpoint, midprice,
sma, tema, trima, wma)
finishedline = flatten(finishedline)
if live:
finishedline = np.array(finishedline, dtype=np.double)
finishedline = finishedline[~np.isnan(finishedline)] # remove nans
else:
finishedline = np.array(finishedline, dtype=np.double)
finishedline = finishedline[~np.isnan(finishedline)]
# print(pid + " has finished")
if len(finishedline) != 326: break
return finishedline
break