def runLoadDailyData(self, evt):
datastore = get_datastore()
dailydatato = datetime.now().replace(tzinfo=pytz.timezone('US/Eastern'))
# if before 6pm, let us stop at previous day - not sure if today is ready to load
if dailydatato.hour < 18:
dailydatato = dailydatato - timedelta(days=1)
dailydataq = Queue()
for ticker in datastore.tickers():
dailydataq.put(ticker)
datastore.close()
# for sending status messages back to the ui
statusq = Queue()
progressdlgthread = ProgressDialogThread(statusq, self, len(dailydataq))
progressdlgthread.daemon = True
progressdlgthread.start()
num_threads = NUM_THREADS
if not datastore.is_threadsafe():
num_threads = 1
for i in range(num_threads):
t = DailyDataWorkerThread(dailydataq, dailydatato, statusq)
t.daemon=True
t.start()
dailydataq.join()
statusq.put("__DONE__")
statusq.join()
datastore = get_datastore()
datastore.setDataThrough(dailydatato)
datastore.close()
def __init__(self, settings):
DailyTrigger.__init__(self, settings)
# this is a hack until we implement subscribing to multiple streams of data
self.uptrendcheck = dict()
# if True, we are filtering on uptrending SPY, if false, downtrending SPY
# we are using the PREVIOUS bar data, hence the use of HistoricMetric.
# we do not now if TODAY will close above/below the ma before we enter our
# day trades. An alternative would have been to use the open
ticker = settings.get("TrendFilterDailyTrigger", "ticker")
self.uptrend = settings.getboolean("TrendFilterDailyTrigger", "uptrend")
period = settings.getint("TrendFilterDailyTrigger", "period")
close = AdjustedClose()
lastclose = HistoricMetric(close, 1)
ma = SimpleMovingAverage(metric=close, period=period)
lastma = HistoricMetric(ma, 1)
fromdt = datetime(1900, 1, 1)
todt = datetime.now()
datastore = datastorefactory.get_datastore()
for dd in datastore.getDailyData(ticker, fromdt, todt):
close.handle(dd)
lastclose.handle(dd)
ma.handle(dd)
lastma.handle(dd)
if lastma.ready() and lastclose.ready():
if lastclose.value() > lastma.value():
self.uptrendcheck[dd.date] = True
if lastclose.value() < lastma.value():
self.uptrendcheck[dd.date] = False
def generatePAndLPlot(self):
plotter = PAndLPlotter(self.settings)
datastore = datastorefactory.get_datastore()
account = Account(self.settings)
positionSizer = PercentOfEquityPositionSizer(self.settings)
outfile = "equitycurve.csv"
plotter.plot(datastore, account, positionSizer, self.trades, outfile)
# TODO try/finally
datastore.close()
def runLoadTickers(self, evt):
datastore = get_datastore()
#igtl = InvestorGuideTickerLoader()
ntl = NasdaqSecurityLoader()
for ticker in ntl.load_securities():
datastore.addStockBatch(ticker, 0, 0)
#loadstocks.loadstocks(datastore)
datastore.commitBatch()
datastore.close()
def run(self):
Thread.run(self)
datastore = get_datastore()
try:
while True:
ticker = self.queue.get_nowait()
#datastore.
self.queue.task_done()
except Empty:
pass
datastore.close()
def runLoadIntradayData(self, evt):
datastore = get_datastore()
intradaydataq = Queue()
for ticker in datastore.tickers():
if ticker.find(".") < 0:
intradaydataq.put(ticker)
#for i in range(10):
t = IntradayWorkerThread(intradaydataq)
t.daemon=True
t.start()
intradaydataq.join()
datastore.close()
def findsetups(self, fromdt, todt):
""" Run the strategy.
This generates the input and output queues and spawns the threads
to test the strategy.
"""
try:
datastore = datastorefactory.get_datastore()
stocks = self._getTickers(fromdt, datastore)
datastore.close()
print "handling watchlist with %d tickers" % (len(stocks))
queue = Queue()
tradeQueue = Queue()
for stock in stocks:
queue.put_nowait(stock)
if self.numThreads > 1:
for i in range(self.numThreads):
t = StrategyThread(datastorefactory.get_datastore(), self, queue, tradeQueue, fromdt, todt)
t.daemon = True
t.start()
queue.join()
else:
# only configured for 1 thread, so do it inline
t = StrategyThread(datastorefactory.get_datastore(), self, queue, tradeQueue, fromdt, todt)
t.daemon = True
t.run()
try:
while True:
trade = tradeQueue.get_nowait()
self.tradeManager.addTrade(trade)
except Empty:
pass
return self.tradeManager.getStats()
except Exception, e:
print "Error during backtest, aborted"
print e
raise
def run(self):
Thread.run(self)
try:
while True:
ticker = self.queue.get_nowait()
try:
self.statusq.put("Loading %s" % ticker)
datastore = get_datastore()
bardataloader.load(datastore, ticker, self.todate)
datastore.close()
except TypeError:
pass
self.queue.task_done()
except Empty:
pass
print "queue empty, exiting worker"
def run(self):
Thread.run(self)
datastore = get_datastore()
try:
while True:
ticker = self.queue.get_nowait()
try:
intradayloader.load(datastore, ticker, 60)
intradayloader.load(datastore, ticker, 300)
except TypeError:
pass
except Exception:
pass
self.queue.task_done()
except Empty:
pass
print "queue empty, exiting worker"
datastore.close()
def getStatsNew(self):
datastore = PreloadedProxyDataStore(datastorefactory.get_datastore())
try:
maxValidRStr = self.settings.get("TradeManager", "maxValidR")
except:
maxValidRStr = "None"
if maxValidRStr == "None":
maxValidR = None
else:
maxValidR = float(maxValidRStr)
numtrades = float(len(self.trades))
numwins = 0.0
totalpandl = 0.0
totalabspandl = 0.0
totalwin = 0.0
totalloss = 0.0
timeintrade=0
timeinwin=0
timeinloss=0
rs = list()
drawdownsizes = dict()
drawdownsizes["5%"] = None
drawdownsizes["10%"] = None
drawdownsizes["25%"] = None
drawdownsizes["25%"] = None
drawdownsizes["50%"] = None
drawdownsizes["75%"] = None
drawdownsizes["90%"] = None
drawdownsizes["optimal f"] = None
leverage_stats = list()
self.trades = sorted(self.trades, key=lambda trade: trade.getEntryDate())
best_final_equity = 0
for risk in (0.005, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05, 0.06, 0.07, 0.08, 0.09, 0.1, \
0.12, 0.14, 0.15, 0.16, 0.18, 0.2, 0.25, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1.0, \
1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 4.0,):
trades = copy.deepcopy(self.trades)
account = Account(self.settings)
max_equity = account.cash
posSizer = PercentOfEquityPositionSizer(self.settings, risk, 4.0, 10)
pandlPlotter = PAndLPlotter(self.settings)
equityCurve = pandlPlotter.generate_equity_curve(datastore, account, \
posSizer, trades)
max_drawdown = 0
final_equity = 0
for data in equityCurve:
equity = data["equity"]
if equity < max_equity:
drawdown = (max_equity-equity)/max_equity
if drawdown > max_drawdown:
max_drawdown = drawdown
if equity > max_equity:
max_equity = equity
final_equity = equity
leverage_stat = dict()
leverage_stat["risk"] = risk
leverage_stat["drawdown"] = max_drawdown
leverage_stat["final_equity"] = final_equity
leverage_stats.append(leverage_stat)
keys = list()
if final_equity > best_final_equity:
keys.append("optimal f")
best_final_equity = equity
if max_drawdown < .9:
keys.append("90%")
if max_drawdown < .75:
keys.append("75%")
if max_drawdown < .5:
keys.append("50%")
if max_drawdown < 0.25:
keys.append("25%")
if max_drawdown < 0.1:
keys.append("10%")
if max_drawdown < 0.05:
keys.append("5%")
for key in keys:
if drawdownsizes[key] == None or drawdownsizes[key][1] < final_equity:
drawdownsizes[key] = (risk, final_equity)
for trade in self.trades:
pandl = trade.pandlR()
if pandl == 0:
pandl = 0
rs.append(0)
abspandl = 0
else:
# disregard outrageous R values, per max valid R setting
if maxValidR == None or pandl <= maxValidR:
rs.append(pandl)
abspandl = trade.pandlPercent()
# disregard outrageous R values, per max valid R setting
if maxValidR == None or pandl <= maxValidR:
totalpandl = totalpandl + pandl
totalabspandl = totalabspandl + abspandl
if pandl > 0:
numwins = numwins + 1.0
totalwin = totalwin + pandl
timeinwin = timeinwin + (trade.time_in_trade()/60)
else:
totalloss = totalloss - pandl
timeinloss = timeinloss + (trade.time_in_trade()/60)
timeintrade = timeintrade + (trade.time_in_trade()/60)
if numtrades > 0:
averageWin=0
averageLoss=0
averageDuration=0
averageWinDuration=0
averageLossDuration=0
if numwins > 0:
averageWin = (totalwin/numwins)
averageWinDuration = timeinwin/(60.0*24.0*float(numwins))
if numwins<numtrades:
averageLoss = (totalloss/(numtrades-numwins))
averageLossDuration = timeinloss/(60.0*24.0*float(numtrades-numwins))
if numtrades > 0:
averageDuration = timeintrade/(60.0*24.0*float(numtrades))
rdev = numpy.std(rs)
# TODO move this somewhere sensible
self.generatePAndLPlot()
self.generateTradesCSV()
return Stats(numtrades,(totalabspandl/numtrades), (totalpandl/numtrades), (numwins/numtrades), \
averageWin = averageWin, averageLoss = averageLoss, averageDuration=averageDuration, \
averageWinDuration=averageWinDuration, averageLossDuration=averageLossDuration, rdev=rdev, \
drawdownsizes = drawdownsizes,
leverage_stats = leverage_stats)
else:
return Stats(0,0.0,0.0,0.0,0.0,0.0,0.0,0.0,0.0, 0.0)
# Program to build list of intersting stocks over a range of days
#
# Interesting stocks are either gapping or had unusual volume on
# the day.
from datetime import timedelta
from strategy import Strategy
from indicators.indicators import SimpleMovingAverage, Close, High, Highest
from strategy import Trade, TradeManager
from watchlist import APRATRVolumePriceScan
from stocklib.datastore import datastorefactory
# https://www.youtube.com/watch?v=nUDBrLOuk44
# he is vague on the daily chart check, I will systemize it with 50sma daily as a trend indicator - I did see he had that on his chart
datastore = datastorefactory.get_datastore()
class Setup:
def __init__(self,date,stock):
self.date=date
self.stock=stock
class GapAfternoonBreakdownStrategy(Strategy):
def __init__(self, settings, watchlist, tradeManager):
Strategy.__init__(self, watchlist, tradeManager)
self.minprice = settings.getfloat("Strategy", "minprice")
self.minvolume = settings.getint("Strategy", "minvolume")
self.mingap = settings.getfloat("Strategy", "mingap")
self.dailysmatrendfilter = self._getintsetting(settings,"Strategy", "dailysmatrendfilter")
self.target = self._getfloatsetting(settings, "Strategy", "target")
def findsetups(self, fromdt, todt):
datastore = datastorefactory.get_datastore()
# stocks = datastore.filterStocksByAvgVolume(fromdt, minvolume)
stocks = self._getTickers(fromdt, datastore)
for stock in stocks:
dailydata = list()
volume = Volume()
avgvolume = SimpleMovingAverage(metric=volume, period=21)
dailyatr = ATR(20)
atrfromdt = fromdt - timedelta(days=max(self.duration, 40)) # 40 to give the atr time to normalize
dailydataiter = iter(datastore.getDailyData(stock, atrfromdt, todt))
dailydataday = None
try:
while dailydataday == None or dailydataday.date < fromdt:
dailydataday = dailydataiter.next()
# have to fix it to a real object for the atr
dailyatr.handle(dailydataday)
dailydata.append(dailydataday)
except StopIteration:
pass
if len(dailydata) > self.duration:
dailydata = dailydata[len(dailydata) - self.duration :]
# ok, we find the highest high and lowest low first
high = 0
low = None
for ddhighfinder in dailydata:
if high < ddhighfinder.high:
high = ddhighfinder.high
if low == None or ddhighfinder.low < low:
low = ddhighfinder.low
# great, now we find how many lower highs are within the mush factor
atrmush = 0
if dailyatr.value() != None:
atrmush = dailyatr.value() * self.mushinessatr
taps = 0
shorttaps = 0
for ddtapfinder in dailydata:
delta = high - ddtapfinder.high
if delta <= atrmush or delta <= self.mushinessfixed:
taps = taps + 1
shortdelta = ddtapfinder.low - low
if shortdelta <= atrmush or delta <= self.mushinessfixed:
shorttaps = shorttaps + 1
# ok, now we can add the next dd - we go ahead and prep some things for the next loop pass
# since we are no longer using them now
for dailydataday in dailydataiter:
saveatr = dailyatr.value()
volume.handle(dailydataday)
avgvolume.handle(dailydataday)
dailyatr.handle(dailydataday)
dailydata.append(dailydataday)
dailydata = dailydata[1:]
trade = None
# as a hack, now we can check our peek ahead and see for free if we
# ever broke the high today. If not, we are done
if (
self.doLongs
and taps >= self.numtaps
and dailydataday.high > high
and high >= self.minprice
and (self.maxprice == None or high <= self.maxprice)
and avgvolume.ready()
and avgvolume.value() >= self.minavgvolume
and (
self.minAPR == None
or (dailyatr.ready() and dailyatr.value() / dailydataday.adjustedClose) >= self.minAPR
)
):
# ok, we need to scan the day
low = None
donchlow = None
if self.donchianstop != None:
low = Low()
donchlow = Lowest(low, self.donchianstop)
intrafromdt = dailydataday.date
intratodt = intrafromdt + timedelta(hours=24)
intradaydata = datastore.getIntradayData(stock, self.period, intrafromdt, intratodt)
if intradaydata != None and len(intradaydata) > 1:
intradaybar = intradaydata[0]
intralow = intradaybar.low
intrahigh = intradaybar.high
taps = 1
for i in range(1, len(intradaydata)):
intradaybar = intradaydata[i]
if trade == None and (
self.maxintradayrangeatr == None
or (saveatr != None and (intrahigh - intralow) < (saveatr * self.maxintradayrangeatr))
):
intralow = min(intralow, intradaybar.low)
if (
intradaybar.high <= intrahigh
and (intrahigh - intradaybar.high) <= self.mushinessfixed2m
):
taps = taps + 1
if intradaybar.high > intrahigh:
if (
taps >= self.taps2m
and intrahigh >= high
and (
self.maxhour == None
or intradaybar.date.hour < self.maxhour
or (intradaybar.date.hour == self.maxhour and intradaybar.date.minute == 0)
)
and (self.minhour == None or intradaybar.date.hour >= self.minhour)
):
# trade entry
if donchlow != None and donchlow.ready():
stop = donchlow.value() - 0.01
else:
stop = intralow - 0.01
entryPrice = min(intradaybar.open, intrahigh + 0.01)
if entryPrice > stop:
trade = Trade(
stock=stock,
entry=intradaybar.date,
entryPrice=min(intradaybar.open, intrahigh + 0.01),
stop=stop,
)
if self.target:
trade.target = trade.entryPrice + (
self.target * (trade.entryPrice - trade.stop)
)
else:
# need to recalculate taps off this new high as we had no signal yet
intrahigh = intradaybar.high
taps = 1
for j in range(0, i - 1):
if (intrahigh - intradaydata[j].high) < self.mushinessfixed2m:
taps = taps + 1
if trade and trade.exit == None:
if intradaybar.low < trade.trailingstop:
# taken out
trade.exit = intradaybar.date
trade.exitPrice = min(intradaybar.open, trade.trailingstop)
if trade.target != None and intradaybar.high > trade.target:
trade.exit = intradaybar.date
trade.exitPrice = max(intradaybar.open, trade.target)
if low != None:
low.handle(intradaybar)
if donchlow != None:
donchlow.handle(intradaybar)
if trade != None and trade.exit == None:
trade.exit = intradaybar.date
trade.exitPrice = intradaybar.close
if trade:
self.tradeManager.addTrade(trade)
trade = None
trade = None
# SHORTS
# as a hack, now we can check our peek ahead and see for free if we
# ever broke the low today. If not, we are done
if (
self.doShorts
and shorttaps >= self.numtaps
and dailydataday.low < low
and low >= self.minprice
and avgvolume.ready()
and avgvolume.value() >= self.minavgvolume
and (
self.minAPR == None
or (dailyatr.ready() and dailyatr.value() / dailydataday.adjustedClose) >= self.minAPR
)
):
# ok, we need to scan the day
high = None
donchhigh = None
if self.donchianstop != None:
high = High()
donchhigh = Highest(high, self.donchianstop)
intrafromdt = dailydataday.date
intratodt = intrafromdt + timedelta(hours=24)
intradaydata = datastore.getIntradayData(stock, 300, intrafromdt, intratodt)
if intradaydata != None and len(intradaydata) > 1:
intradaybar = intradaydata[0]
intralow = intradaybar.low
intrahigh = intradaybar.high
taps = 1
for i in range(1, len(intradaydata)):
intradaybar = intradaydata[i]
if trade == None and (
self.maxintradayrangeatr == None
or (saveatr != None and (intrahigh - intralow) < (saveatr * self.maxintradayrangeatr))
):
intrahigh = max(intrahigh, intradaybar.high)
if (
intradaybar.low >= intralow
and (intradaybar.low - intralow) <= self.mushinessfixed2m
):
taps = taps + 1
if intradaybar.low < intralow:
if (
taps >= self.taps2m
and intralow <= low
and (
self.maxhour == None
or intradaybar.date.hour < self.maxhour
or (intradaybar.date.hour == self.maxhour and intradaybar.date.minute == 0)
)
and (self.minhour == None or intradaybar.date.hour >= self.minhour)
):
# trade entry
if donchhigh != None and donchhigh.ready():
stop = donchhigh.value() + 0.01
else:
stop = intrahigh + 0.01
entryPrice = min(intradaybar.open, intralow - 0.01)
if entryPrice < stop:
trade = Trade(
stock=stock, entry=intradaybar.date, entryPrice=entryPrice, stop=stop
)
if self.target:
trade.target = trade.entryPrice - (
self.target * (trade.stop - trade.entryPrice)
)
else:
# need to recalculate taps off this new high as we had no signal yet
intralow = intradaybar.low
taps = 1
for j in range(0, i - 1):
if (intralow - intradaydata[j].low) < self.mushinessfixed2m:
taps = taps + 1
if trade and trade.exit == None:
if intradaybar.high >= trade.trailingstop:
# taken out
trade.exit = intradaybar.date
trade.exitPrice = max(intradaybar.open, trade.trailingstop)
if trade.target != None and intradaybar.low < trade.target:
trade.exit = intradaybar.date
trade.exitPrice = min(intradaybar.open, trade.target)
if high != None:
high.handle(intradaybar)
if donchhigh != None:
donchhigh.handle(intradaybar)
if trade != None and trade.exit == None:
trade.exit = intradaybar.date
trade.exitPrice = intradaybar.close
if trade:
self.tradeManager.addTrade(trade)
trade = None
trade = None
# redo daily setup for the next day, already loaded in above the intraday loop
# ok, we find the highest high first
high = 0
low = None
for ddhighfinder in dailydata:
if high < ddhighfinder.high:
high = ddhighfinder.high
if low == None or ddhighfinder.low < low:
low = ddhighfinder.low
# great, now we find how many lower highs are within the mush factor
atrmush = 0
if dailyatr.value() != None:
atrmush = dailyatr.value() * self.mushinessatr
taps = 0
shorttaps = 0
for ddtapfinder in dailydata:
delta = high - ddtapfinder.high
shortdelta = ddtapfinder.low - low
if delta <= atrmush or delta <= self.mushinessfixed:
taps = taps + 1
if shortdelta <= atrmush or shortdelta <= self.mushinessfixed:
shorttaps = shorttaps + 1
return self.tradeManager.getStats()
