def modify_dfc(c):
"""Edit or append a single index to global candle dataframe.
@c: candle dict
"""
pair = c['pair']
freq = strtofreq(c['freqstr'])
open_time = pd.Timestamp(c['open_time'].replace(tzinfo=None))
index = (pair, freq, open_time)
# Modify existing DF index.
if index in app.bot.dfc.index:
try:
app.bot.dfc.ix[index] = [c[n] for n in columns[3:]]
except Exception as e:
log.debug(str(e))
log.debug("candle: {}".format(c))
log.debug("app.bot.dfc.ix: {}".format(app.bot.dfc.ix[index]))
# Create index in new DF and append.
else:
c_ = c.copy()
c_['freq'] = strtofreq(c_['freqstr'])
c_['open_time'] = open_time
c_ = { k:v for k,v in c_.items() if k in columns}
df = pd.DataFrame.from_dict([c_], orient='columns')\
.set_index(['pair','freq','open_time'])
app.bot.dfc = app.bot.dfc.append(df)
app.bot.dfc = app.bot.dfc.sort_index()
def macd_med_trend_filter():
TRD_FREQS = docs.botconf.TRD_FREQS
DEF_KLINE_HIST_LEN = docs.botconf.DEF_KLINE_HIST_LEN
for pair in app.bot.get_pairs():
for freqstr in TRD_FREQS:
freq = strtofreq(freqstr)
periods = int((strtoms("now utc") - strtoms(DEF_KLINE_HIST_LEN)) / ((freq * 1000))) #/2))
df = app.bot.dfc.loc[pair,freq]
dfh, phases = macd.histo_phases(df, pair, freqstr, periods)
# Format for log output
dfh = dfh.tail(3)
idx = dfh.index.to_pydatetime()
dfh.index = [ to_local(n.replace(tzinfo=pytz.utc)).strftime("%m-%d %H:%M") for n in idx]
dfh = dfh.rename(columns={
'ampMean':'amp.mean',
'ampMax':'amp.max',
'priceY':'price.y',
'priceX':'price.x'
})
scanlog("{} {} Macd Phases".format(pair, freqstr))
lines = dfh.to_string(
columns=['bars','amp.mean','amp.max','price.y','price.x','capt'],
formatters={
'bars': ' {:} '.format,
'amp.mean': ' {:+.2f}'.format,
'amp.max': ' {:+.2f}'.format,
'price.y': ' {:+.2f}%'.format,
'price.x': ' {:+.2f}%'.format,
'capt': ' {:.2f}'.format
}
).split("\n")
[ scanlog(line) for line in lines]
scanlog("")
def query_api(pair, freqstr, startstr=None, endstr=None):
"""Get Historical Klines (candles) from Binance.
@freqstr: 1m, 3m, 5m, 15m, 30m, 1h, ettc
"""
client = app.bot.client
t1 = Timer()
ms_period = strtofreq(freqstr) * 1000
end = strtoms(endstr or "now utc")
start = strtoms(startstr or DEF_KLINE_HIST_LEN)
results = []
while start < end:
try:
data = client.get_klines(
symbol=pair,
interval=freqstr,
limit=BINANCE_REST_QUERY_LIMIT,
startTime=start, endTime=end)
except Exception as e:
log.exception("Binance API request error. e=%s", str(e))
continue
if len(data) == 0:
start += ms_period
else:
results += data
start = data[-1][0] + ms_period
log.debug('%s %s %s queried [%ss].', len(results), freqstr, pair,
t1.elapsed(unit='s'))
return results
def trades(trade_ids):
db = app.get_db()
cols = [
'freq', "type", "Δprice", "macd", "rsi", "zscore", "time", "algo",
"details"
]
data, indexes = [], []
for _id in trade_ids:
record = db.trades.find_one({"_id": _id})
indexes.append(record['pair'])
ss1 = record['snapshots'][0]
ss_new = record['snapshots'][-1]
df = app.bot.dfc.loc[record['pair'],
strtofreq(record['freqstr'])].tail(100)
if len(record['orders']) > 1:
c1 = ss1['candle']
c2 = ss_new['candle']
data.append([
c2['freqstr'], 'SELL',
pct_diff(c1['close'],
c2['close']), ss_new['indicators']['macd']['value'],
ss_new['indicators']['rsi'], ss_new['indicators']['zscore'],
to_relative_str(now() - record['start_time']), record['algo'],
record['details'][-1]['section'].title()
])
# Buy trade
else:
c1 = ss1['candle']
data.append([
c1['freqstr'], 'BUY', 0.0,
ss_new['indicators']['macd']['value'],
ss_new['indicators']['rsi'], ss_new['indicators']['zscore'],
"-", record['algo'], record['details'][-1]['section'].title()
])
if len(data) == 0:
return tradelog("0 trades executed")
df = pd.DataFrame(data, index=pd.Index(indexes), columns=cols)
df = df[cols]
lines = df.to_string(
formatters={
cols[0]: ' {}'.format,
cols[1]: ' {}'.format,
cols[2]: ' {:+.2f}%'.format,
cols[3]: ' {:+.3f}'.format,
cols[4]: '{:.0f}'.format,
cols[5]: '{}'.format,
cols[5]: '{}'.format,
cols[6]: '{}'.format
}).split("\n")
tradelog('-' * TRADELOG_WIDTH)
tradelog("{} trade(s) executed:".format(len(df)))
[tradelog(line) for line in lines]
def sma_med_trend_filter():
"""Identify pairs in intermediate term uptrend via 1d SMA slope. Enable
each filtered pair in real-time + load its historic data into memory.
"""
################################################################
# TODO: Repeat on 1h after 1d to filter out recent dips.
################################################################
n_candles = len(app.bot.dfc)
trend = docs.botconf.TRD_PAIRS['midterm']
lbl = "sma{}_slope".format(trend['span'])
freq = strtofreq(trend['freqstr'])
filtered = trend['filters'][0](tickers.binance_24h())
results = []
for pair in filtered:
bulk_append_dfc(api_update([pair], [trend['freqstr']], silent=True))
sma = app.bot.dfc.loc[pair, freq]['close']\
.rolling(trend['span']).mean().pct_change()*100
if all([fn(sma) for fn in trend['conditions']]):
set_pairs([pair], 'ENABLED')
results.append({
'pair': pair,
lbl: sma.iloc[-1]
})
time.sleep(3)
df = pd.DataFrame(results)\
.set_index('pair').sort_values(lbl).round(1)
lines = df.to_string(
columns=[lbl],
formatters={lbl:'{:+.1f}%'.format}
).split("\n")
[scanlog(line) for line in lines]
scanlog("")
lock.acquire()
print("Scanner thread: sma_med_trend completed.\n"\
"{} trading pairs enabled.\n "\
"{:+,} historic candles loaded."\
.format(len(get_pairs()), len(app.bot.dfc) - n_candles))
lock.release()
return df
def positions():
"""Position summary.
"""
db = app.get_db()
cols = ["freq", "price", "Δprice", "macd", "rsi", "zscore", "time", "algo"]
data, indexes = [], []
opentrades = db.trades.find({'status': 'open'})
for record in opentrades:
ss1 = record['snapshots'][0]
c1 = ss1['candle']
ss_new = record['snapshots'][-1]
freq = strtofreq(record['freqstr'])
df = app.bot.dfc.loc[record['pair'], freq]
dfmacd, phases = macd.histo_phases(df, record['pair'],
record['freqstr'], 100)
data.append([
c1['freqstr'], df.iloc[-1]['close'],
pct_diff(c1['close'], df.iloc[-1]['close']), phases[-1].iloc[-1],
signals.rsi(df['close'], 14),
signals.zscore(df['close'], df.iloc[-1]['close'], 21),
to_relative_str(now() - record['start_time']), record['algo']
])
indexes.append(record['pair'])
if opentrades.count() == 0:
tradelog("0 open positions")
else:
df = pd.DataFrame(data, index=pd.Index(indexes), columns=cols)
df = df[cols]
lines = df.to_string(
formatters={
cols[0]: ' {}'.format,
cols[1]: ' {:g}'.format,
cols[2]: ' {:+.2f}%'.format,
cols[3]: ' {:+.3f}'.format,
cols[4]: '{:.0f}'.format,
cols[5]: '{}'.format,
cols[6]: ' {}'.format
}).split("\n")
tradelog('-' * TRADELOG_WIDTH)
tradelog("{} position(s):".format(len(df)))
[tradelog(line) for line in lines]
return df
def bulk_append_dfc(candlelist):
"""Append multiple indexes to global candle dataframe.
@candles: list of candle dicts
"""
candles_ = []
# Rebuild candle list formatted for dataframe.
for c in candlelist:
c_ = c.copy()
c_['freq'] = strtofreq(c_['freqstr'])
c_['open_time'] = pd.Timestamp(c_['open_time'].replace(tzinfo=None))
c_ = { k:v for k,v in c_.items() if k in columns}
candles_.append(c_)
df = pd.DataFrame.from_dict(candles_, orient='columns')\
.set_index(['pair','freq','open_time'])
app.bot.dfc = app.bot.dfc.append(df).sort_index()
# Drop any rows that have duplicate (pair,freq,open_time) indexes.
app.bot.dfc = app.bot.dfc[~app.bot.dfc.index.duplicated(keep='first')]
app.bot.dfc = app.bot.dfc.sort_index()
return app.bot.dfc
def snapshot(c):
"""Gather state of trade--candle, indicators--each tick and save to DB.
"""
global dfW
book = None
wick_slope = macd_value = amp_slope = np.nan
pair, freqstr = c['pair'], c['freqstr']
buyratio = (c['buy_vol'] / c['volume']) if c['volume'] > 0 else 0.0
# MACD Indicators
dfm_dict = {}
df = app.bot.dfc.loc[pair, strtofreq(freqstr)]
try:
dfmacd, phases = macd.histo_phases(df,
pair,
freqstr,
100,
to_bson=True)
except Exception as e:
lock.acquire()
print('snapshot exc')
print(str(e))
lock.release()
if len(dfmacd) < 1:
dfm_dict['bars'] = 0
else:
dfm_dict = dfmacd.iloc[-1].to_dict()
dfm_dict['bars'] = int(dfm_dict['bars'])
macd_value = phases[-1].iloc[-1]
amp_slope = phases[-1].diff().ewm(span=min(3, len(phases[-1])),
min_periods=0).mean().iloc[-1]
if c['closed']:
# Find price EMA WITHIN the wick (i.e. each trade). Very
# small movements.
#prices = dfW.loc[c['pair'], c['freqstr']]['close']
#wick_slope = prices.diff().ewm(span=len(prices)).mean().iloc[-1]
# FIXME
wick_slope = 0.0
return {
'pair': pair,
'time': now(),
'book': None,
'candle': c,
'indicators': {
'buyRatio': round(buyratio, 2),
'rsi': signals.rsi(df['close'].tail(100), 14),
'wickSlope': wick_slope,
'zscore': signals.zscore(df['close'], c['close'], 21),
'macd': {
**dfm_dict,
**{
'ampSlope': round(amp_slope, 2),
'value': round(macd_value, 2)
}
}
}
}
def histo_hist(df, pair, freqstr, startstr, periods):
df = df.loc[pair, strtofreq(freqstr)]
return macd.histo_phases(df, pair, freqstr, periods)
def histo_phases(df, pair, freqstr, periods, to_bson=False):
"""Groups and analyzes the MACD histogram phases within given timespan.
Determines how closely the histogram bars track with price.
"""
DF = pd.DataFrame
freq = strtofreq(freqstr)
df = df.copy()
dfmacd = generate(df).tail(periods)['macd_diff']
np_arr, descs, phases = [], [], []
idx = 0
while idx < len(dfmacd):
try:
iloc, row, phase, desc = next_phase(dfmacd, freq, idx)
except Exception as e:
log.info("{}".format(pair))
pprint(dfmacd)
dfmacd = dfmacd #.drop_duplicates()
if row is None:
idx += 1
continue
else:
np_arr.append(row)
descs.append(desc)
phases.append(phase)
idx = iloc[1] + 1
dfh = DF(np_arr,
columns=['start', 'end', 'bars', 'phase', 'ampMean', 'ampMax'])
# Gen labels and calc % price changes
lbls, pxy_corr, pct_py, pct_px = [], [], [], []
j = 0
for i in range(0, len(dfh)):
if j == len(abc):
j = 0
lbls.append("{} ({})".format(abc[j].upper(), dfh.iloc[i]['phase']))
j += 1
# Determine correlation between histogram bars and price movement
# Find overall histogram=>candle close correlation
for i in range(0, len(dfh)):
_slice = df.loc[slice(dfh.iloc[i]['start'], dfh.iloc[i]['end'])]
py = pct_diff(_slice['low'].min(), _slice['high'].max())
if dfh['ampMean'].iloc[i] < 0:
py *= -1
pct_py.append(py)
pct_px.append(
pct_diff(_slice.iloc[0]['open'], _slice.iloc[-1]['close']))
if len(phases[i]) == len(_slice):
pxy_corr.append(phases[i].corr(_slice['close']))
else:
pxy_corr.append(np.nan)
dfh['lbl'] = lbls
dfh['duration'] = dfh['end'] - dfh['start']
dfh['priceY'] = pct_py
dfh['priceX'] = pct_px
dfh['capt'] = abs(dfh['priceX'] / dfh['priceY'])
dfh['corr'] = pxy_corr
# Append cols/clean up formatting
dfh.index = dfh['start']
dfh = dfh.sort_index()
dfh = dfh[[
'lbl', 'bars', 'duration', 'ampMean', 'ampMax', 'priceY', 'priceX',
'capt', 'corr'
]].round(2)
if to_bson:
dfh = dfh.reset_index()
dfh['start'] = [
str(to_local(n.to_pydatetime().replace(tzinfo=pytz.utc)))
for n in dfh['start']
]
dfh['duration'] = dfh['duration'].apply(
lambda x: str(x.to_pytimedelta()))
dfh['bars'].astype('int')
return (dfh, phases)
def plot(pairs=None,
freqstr=None,
trades=None,
startstr=None,
indicators=None,
normalize=False):
'''Generate plotly chart html file.
Stacked Subplots with a Shared X-Axis
'''
from app.bot.candles import api_update, bulk_load, bulk_append_dfc
db = app.db
startdt = None
annotations, indicators, traces, indices = [], [], [], []
def_start = strtodt(startstr or DEF_KLINE_HIST_LEN)
if trades is not None:
indices = [(n['pair'], strtofreq(n['freqstr'])) for n in trades]
startdt = min([n['start_time'] for n in trades] + [def_start])
else:
indices = [(n, strtofreq(freqstr)) for n in pairs]
startdt = def_start
# Price traces.
for idx in set(indices):
if idx not in app.bot.dfc.index:
bulk_load([idx[0]], [freqtostr(idx[1])], startdt=startdt)
if idx not in app.bot.dfc.index:
bulk_append_dfc(api_update([idx[0]], [freqtostr(idx[1])]))
df = app.bot.dfc.ix[idx[0:2]]
traces.append(
go.Scatter(
x=df.index,
y=signals.normalize(df['close']) if normalize else df['close'],
name="{} {}".format(idx[0], freqtostr(idx[1]))))
# Trade entry/exit annotations
for trade in trades:
yoffset = -20
df = app.bot.dfc.ix[(trade['pair'], strtofreq(trade['freqstr']))]
df_n = signals.normalize(df['close'])
for n in [0, -1]:
ss = trade['snapshots'][n]
loc = df.index.get_loc(ss['candle']['open_time'].astimezone(
pytz.utc)) #.replace(tzinfo=pytz.utc))
annotations.append(
dict(
x=ss['candle']['open_time'].astimezone(
pytz.utc), #tzinfo=pytz.utc),
y=df_n.iloc[loc],
xref='x',
yref='y',
text='{} {}'.format(trade['algo'],
'entry' if n == 0 else 'exit'),
showarrow=True,
arrowhead=7,
ax=0,
ay=yoffset))
print("{} annotations".format(len(annotations)))
# Indicators
for indic in indicators:
'''
dfmacd = generate(df)
t2 = go.Bar(
x=dfmacd.index,
y=dfmacd['macd_diff'],
name="MACD_diff (normalized)",
yaxis='y2')
t3 = go.Bar(
x=dfmacd.index,
y=dfmacd['volume'],
name="Volume",
yaxis='y3')
data = [t1, t2, t3]
'''
pass
n_div = [len(traces) > 0,
len(indicators) > 0].count(True) #len(annotations)>0].count(True)
# Setup axes/formatting
layout = go.Layout(
title = '{} {} Trade Summary (24 Hours)'\
.format(pairs, freqstr),
margin = dict(
l=100, r=100, b=400, t=75, pad=25
),
xaxis = dict(
anchor = "y3",
#domain=[0.0, 0.1],
title="<BR>"
),
yaxis=dict(
domain=[1/n_div, 1]
),
annotations=annotations
#yaxis2=dict(
# domain=[0.2, 0.4]
#),
#yaxis3=dict(
# domain=[0, 0.2]
#),
)
fig = go.Figure(data=traces, layout=layout)
return fig
def bulk_load(pairs, freqstrs, startstr=None, startdt=None):
"""Merge only newly updated DB records into dataframe to avoid ~150k
DB reads every main loop.
"""
db = app.get_db()
t1 = Timer()
columns = ['open', 'close', 'high', 'low', 'trades', 'volume', 'buy_vol']
exclude = ['_id', 'quote_vol','sell_vol', 'close_time']
proj = dict(zip(exclude, [False]*len(exclude)))
query = {
'pair': {'$in':pairs},
'freqstr': {'$in':freqstrs}
}
if startstr:
query['open_time'] = {'$gte':parse(startstr)}
elif startdt:
query['open_time'] = {'$gte':startdt}
batches = db.candles.find_raw_batches(query, proj)
if batches.count() < 1:
print("No db matches for query {}.".format(query))
return app.bot.dfc
dtype = np.dtype([
('pair', 'S12'),
('freqstr', 'S3'),
('open_time', np.int64),
('open', np.float64),
('close', np.float64),
('high', np.float64),
('low', np.float64),
('buy_vol', np.float64),
('volume', np.float64),
('trades', np.int32)
])
# Bulk load mongodb records into predefined, fixed-size numpy array.
# 10x faster than manually casting mongo cursor into python list.
try:
ndarray = sequence_to_ndarray(batches, dtype, batches.count())
except Exception as e:
print(str(e))
return app.bot.dfc
# Build multi-index dataframe from ndarray
df = pd.DataFrame(ndarray)
df['open_time'] = pd.to_datetime(df['open_time'], unit='ms')
df['freqstr'] = df['freqstr'].str.decode('utf-8')
df['pair'] = df['pair'].str.decode('utf-8')
# Convert freqstr->freq to enable index sorting
df = df.rename(columns={'freqstr':'freq'})
[df['freq'].replace(n, strtofreq(n), inplace=True) for n in TRD_FREQS]
df.sort_values(by=['pair','freq','open_time'], inplace=True)
dfc = pd.DataFrame(df[columns].values,
index = pd.MultiIndex.from_arrays(
[df['pair'], df['freq'], df['open_time']],
names = ['pair','freq','open_time']),
columns = columns
).sort_index()
n_bulk = len(dfc)
app.bot.dfc = pd.concat([app.bot.dfc, dfc])
app.bot.dfc = app.bot.dfc[~app.bot.dfc.index.duplicated(keep='first')]
n_merged = len(dfc) - n_bulk
log.debug("{:,} docs loaded, {:,} merged in {:,.1f} ms."\
.format(n_bulk, n_merged, t1))
return app.bot.dfc