def test_bars(length=50):
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strat_name = 'test_%d' % length
strategy = MStrategy(strat_name, strategy_params={'length': length})
strategy.bar_interval = 0
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
porto_name = 'portfolio_%d' % length
portfolio = Portfolio(porto_name, None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
log.info("START JOB %s = %s" % (porto_name, datetime.datetime.now()))
simData = DataFeedBars('20100315.SPY.1m.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
# portfolio.stats(write_data=True)
filename = 'TEST_BAR_X_%d.xls' % length
port_stats = portfolio.stats(filename=filename)
print 'portfolio stats'
pprint(port_stats)
log.info("END JOB %s = %s" % (porto_name, datetime.datetime.now()))
return portfolio
def test_multiple_symbols():
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strategy = MStrategy('test_strategy', strategy_params={'length': 10})
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
portfolio = Portfolio('test_porto', None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
log.info("START JOB= %s" % datetime.datetime.now())
## combined file of SPY, IWM, and QQQQ
simData = DataFeedIntraday('20100315.XXX.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
log.info("STAT JOB= %s" % datetime.datetime.now())
port_stats = portfolio.stats(filename='TESTER.xls')
pprint(port_stats)
log.info("END JOB= %s" % datetime.datetime.now())
log.info("LEN DATA JOB= %s" % simData.count)
def test_retrace_strategy(strategy_params, run_id):
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strat_name = 'test_%04d' % run_id
## NOTE strategy_params is a dict that the strategy
## uses to initialize itself.
strategy = RetraceStrategy(strat_name, strategy_params=strategy_params)
strategy.bar_interval = 1
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
porto_name = 'retrace_%d' % run_id
portfolio = Portfolio(porto_name, None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
simData = DataFeedDaily('daily.SPY.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
return portfolio.stats()
def test_strategy_fills(side):
## handle long and short fills, all possiblities
## strategy queues
latch = DataLatch(1)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strategy = MStrategy('test_strategy', strategy_params={'length': 10})
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
strategy.start()
ts = datetime.datetime(2014, 8, 16, 12, 30, 0)
## build inital position
s1_fill_q.put(Fill('AAPL', 100.00, 100, side, ts, 1))
s1_fill_q.put(Fill('AAPL', 101.50, 50, side, ts, 2))
s1_fill_q.put(
Fill('AAPL', 110.00, 50, side, ts + datetime.timedelta(seconds=10), 3))
rev = Order.SELL
if side == rev: rev = Order.BUY
## take some of off
s1_fill_q.put(
Fill('AAPL', 107.00, 20, rev, ts + datetime.timedelta(seconds=20), 4))
s1_fill_q.put(
Fill('AAPL', 107.00, 20, rev, ts + datetime.timedelta(seconds=25), 5))
## flip position
s1_fill_q.put(
Fill('AAPL', 110.00, 200, rev, ts + datetime.timedelta(seconds=100),
6))
s1_fill_q.put(
Fill('AAPL', 108.50, 10, rev, ts + datetime.timedelta(seconds=110), 7))
s1_fill_q.put(
Fill('AAPL', 106.50, 100, rev, ts + datetime.timedelta(seconds=110),
8))
##close position
s1_fill_q.put(
Fill('AAPL', 109.00, 100, side, ts + datetime.timedelta(seconds=200),
9))
s1_fill_q.put(
Fill('AAPL', 109.00, 50, side, ts + datetime.timedelta(seconds=200),
10))
time.sleep(5)
strategy.shutdown()
strategy.join()
def run(self, datafeed):
log.info("reset_on_EOD = %s" % self.reset_on_EOD)
self.latch = DataLatch(len(self.strategies) + 2)
self.portfolio.latch = self.latch
self.exchange.latch = self.latch
for s in self.strategies:
s.latch = self.latch
bg = datetime.datetime.now()
if self.verbose: log.info('Sim Start: %s' % bg)
for market_data in datafeed:
if market_data != DataFeed.SENTINEL:
self.latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
for s in self.strategies:
s.on_data_sim(market_data)
self.exchange.on_data_sim(market_data)
self.portfolio.on_data_sim(market_data)
else:
if self.reset_on_EOD:
## handle EOD processing
self.portfolio.on_EOD()
for s in self.strategies:
s.on_EOD()
self.exchange.on_EOD()
nd = datetime.datetime.now()
if self.verbose:
log.info('Sim Completed: %s' % nd)
log.info('Time Elapsed: %s' % (nd - bg))
return self.dump()
def test_exchange():
latch = DataLatch(1)
## exchange queues
order_q = DQueue()
fill_q = DQueue()
exchange = Exchange()
exchange.latch = latch
## bind exchange and portfolio together
tester = Tester()
exchange.IN_orders = order_q
exchange.OUT_fills = fill_q
tester.IN_queue = fill_q
exchange.start()
tester.start()
o1 = Order('test', 'AAPL', Order.BUY, 100, Order.MARKET, None, None)
o1.stamp_time(parse_date("20140311"))
order_q.put(o1)
o2 = Order('test', 'AAPL', Order.BUY, 200, Order.MARKET, None, None)
o2.stamp_time(parse_date("20140816"))
order_q.put(o2)
o3 = Order('test', 'AAPL', Order.SELL, 300, Order.MARKET, None, None)
o3.stamp_time(parse_date("20140101"))
order_q.put(o3)
simData = DataFeedDaily('AAPL.csv')
for market_data in simData:
latch.trap(market_data)
exchange.on_data(market_data)
exchange.shutdown()
tester.shutdown()
exchange.join()
tester.join()
def test_strategy_execute():
latch = DataLatch(2)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strategy = MStrategy('test_strategy', strategy_params={'length': 10})
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
exchange = Exchange()
exchange.IN_orders = strategy.OUT_orders
exchange.OUT_fills = strategy.IN_fills
exchange.latch = latch
exchange.start()
strategy.start()
log.info("START JOB= %s" % datetime.datetime.now())
simData = DataFeedIntraday('20100315.SPY.csv')
for market_data in simData:
latch.trap(market_data)
exchange.on_data(market_data)
strategy.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
strategy.shutdown()
exchange.join()
strategy.join()
log.info("END JOB= %s" % datetime.datetime.now())
log.info("LEN DATA JOB= %s" % simData.count)
def test_strategy_order_update():
## do partial fill and update, both sides
latch = DataLatch(1)
s1_order_q = DQueue()
s1_fill_q = DQueue()
ts = datetime.datetime(2014, 8, 16, 12, 30, 0)
strategy = MStrategy('test_strategy', strategy_params={'length': 10})
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
strategy.start()
o1 = Order(strategy.name, 'AAPL', Order.SELL, 100, Order.MARKET, None,
None)
o2 = Order(strategy.name, 'AAPL', Order.SELL, 200, Order.MARKET, None,
None)
p1 = Order(strategy.name, 'AAPL', Order.BUY, 100, Order.MARKET, None, None)
p2 = Order(strategy.name, 'AAPL', Order.BUY, 200, Order.MARKET, None, None)
strategy.send_order(o1)
strategy.send_order(o2)
strategy.send_order(p1)
strategy.send_order(p2)
# allow time for orders to be sent
time.sleep(2)
s1_fill_q.put(Fill('AAPL', 100.00, 100, Order.SELL, ts, o1.order_id))
s1_fill_q.put(
Fill('AAPL', 101.50, 50, Order.SELL, ts, o2.order_id, qty_left=150))
s1_fill_q.put(Fill('AAPL', 104.00, 100, Order.BUY, ts, p1.order_id))
s1_fill_q.put(
Fill('AAPL', 105.50, 70, Order.BUY, ts, p2.order_id, qty_left=130))
time.sleep(2)
strategy.shutdown()
strategy.join()
class Simulator(object):
def __init__(self):
self.latch = None
self.strategies = []
self.portfolio = Portfolio('portfolio', None)
self.exchange = Exchange()
self.portfolio.IN_fills = self.exchange.OUT_fills
self.stats = None
self.verbose = True
## call on_EOD funcs at end of data file
self.reset_on_EOD = True
self.scoring_function = None
def add_strategy(self, strategy):
self.strategies.append(strategy)
strategy.IN_fills = DQueue()
self.portfolio.add(strategy)
self.exchange.add(strategy)
def run(self, datafeed):
log.info("reset_on_EOD = %s" % self.reset_on_EOD)
self.latch = DataLatch(len(self.strategies) + 2)
self.portfolio.latch = self.latch
self.exchange.latch = self.latch
for s in self.strategies:
s.latch = self.latch
bg = datetime.datetime.now()
if self.verbose: log.info('Sim Start: %s' % bg)
for market_data in datafeed:
if market_data != DataFeed.SENTINEL:
self.latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
for s in self.strategies:
s.on_data_sim(market_data)
self.exchange.on_data_sim(market_data)
self.portfolio.on_data_sim(market_data)
else:
if self.reset_on_EOD:
## handle EOD processing
self.portfolio.on_EOD()
for s in self.strategies:
s.on_EOD()
self.exchange.on_EOD()
nd = datetime.datetime.now()
if self.verbose:
log.info('Sim Completed: %s' % nd)
log.info('Time Elapsed: %s' % (nd - bg))
return self.dump()
def dump(self):
if not self.scoring_function:
self.scoring_function = fitness_function
if not self.stats:
self.stats = self.portfolio.stats()
self.stats['_score'] = self.scoring_function(self.stats)
summary = OrderedDict()
header = [
'_score', 'cnt', 'w_pct', 'pr', 'pnl', 'mtm_pnl', 'max_equ',
'max_dd'
]
for k in header:
summary[k] = self.stats[k]
table = PrettyTable(header)
table.add_row(summary.values())
header.pop(1) ## remove 'cnt' label
for k in header:
table.float_format[k] = '0.2'
if self.verbose: log.info('\n%s' % table)
## return stat summary for potential use elsewhere
#return summary
return self.stats
## show the equity curve
def show(self):
if not self.stats:
self.stats = self.portfolio.stats()
curve_df = self.portfolio.storage['curve']['portfolio']
curve_df.plot(x='timestamp')
## write output
def write(self, filename):
root = ".".join(filename.split('.')[:-1])
if not root: root = filename
xls = ".".join([root, 'xls'])
pkl = ".".join([root, 'pkl'])
log.info('writing pickle file: %s' % pkl)
self.portfolio.write(filename=pkl)
self.portfolio.to_excel(filename=xls)