def test_ndict():
nd = ndict({})
# Properties
assert len(nd) == 0
assert nd.keys() == []
assert nd.values() == []
assert list(nd.iteritems()) == []
# Accessors
nd['x'] = 1
assert nd.x == 1
assert nd['x'] == 1
assert nd.get('y') is None
assert nd.get('y', 'fizzpop') == 'fizzpop'
assert 'x' in nd
assert 'y' not in nd
assert 'x' in nd
assert 'y' not in nd
# Mutability
nd2 = ndict({'x': 1})
assert nd2.x == 1
nd2.x = 2
assert nd2.x == 2
# Class isolation
assert '__init__' not in nd
assert '__iter__' not in nd
assert 'x' not in nd.__dict__
assert nd.get('__init__') is None
assert 'x' not in set(dir(nd))
# Comparison
nd2 = nd.copy()
assert id(nd2) != id(nd)
assert nd2 == nd
nd2['z'] = 3
assert nd2 != nd
class ndictlike(object):
x = 1
assert {'x': 1} == nd
assert ndictlike() != nd
# Deletion
del nd['x']
assert 'x' not in nd
assert nd.get('x') is None
for n in xrange(1000):
dt = datetime.utcnow().replace(tzinfo=pytz.utc)
nd2 = ndict({"dt": dt,
"otherdata": "ishere" * 1000,
"maybeanint": 3})
nd2.dt2 = dt
def get_positions(self):
positions = ndict(internal=position_ndict())
for sid, pos in self.positions.iteritems():
cur = pos.to_dict()
positions[sid] = ndict(cur)
return positions
def get_positions(self):
positions = ndict(internal=position_ndict())
for sid, pos in self.positions.iteritems():
cur = pos.to_dict()
positions[sid] = ndict(cur)
return positions
def test_ndict():
nd = ndict({})
# Properties
assert len(nd) == 0
assert nd.keys() == []
assert nd.values() == []
assert list(nd.iteritems()) == []
# Accessors
nd['x'] = 1
assert nd.x == 1
assert nd['x'] == 1
assert nd.get('y') is None
assert nd.get('y', 'fizzpop') == 'fizzpop'
assert 'x' in nd
assert 'y' not in nd
assert 'x' in nd
assert 'y' not in nd
# Mutability
nd2 = ndict({'x': 1})
assert nd2.x == 1
nd2.x = 2
assert nd2.x == 2
# Class isolation
assert '__init__' not in nd
assert '__iter__' not in nd
assert 'x' not in nd.__dict__
assert nd.get('__init__') is None
assert 'x' not in set(dir(nd))
# Comparison
nd2 = nd.copy()
assert id(nd2) != id(nd)
assert nd2 == nd
nd2['z'] = 3
assert nd2 != nd
class ndictlike(object):
x = 1
assert {'x': 1} == nd
assert ndictlike() != nd
# Deletion
del nd['x']
assert 'x' not in nd
assert nd.get('x') is None
for n in xrange(1000):
dt = datetime.utcnow().replace(tzinfo=pytz.utc)
nd2 = ndict({"dt": dt, "otherdata": "ishere" * 1000, "maybeanint": 3})
nd2.dt2 = dt
def as_portfolio(self):
"""
The purpose of this method is to provide a portfolio
object to algorithms running inside the same trading
client. The data needed is captured raw in a
PerformancePeriod, and in this method we rename some
fields for usability and remove extraneous fields.
"""
portfolio = self.__core_dict()
# rename:
# ending_cash -> cash
# period_open -> backtest_start
#
# remove:
# period_close, starting_value,
# cumulative_capital_used, max_leverage, max_capital_used
portfolio['cash'] = portfolio['ending_cash']
portfolio['start_date'] = portfolio['period_open']
portfolio['positions_value'] = portfolio['ending_value']
del(portfolio['ending_cash'])
del(portfolio['period_open'])
del(portfolio['period_close'])
del(portfolio['starting_value'])
del(portfolio['ending_value'])
del(portfolio['cumulative_capital_used'])
del(portfolio['max_leverage'])
del(portfolio['max_capital_used'])
portfolio['positions'] = self.get_positions()
return ndict(portfolio)
def as_portfolio(self):
"""
The purpose of this method is to provide a portfolio
object to algorithms running inside the same trading
client. The data needed is captured raw in a
PerformancePeriod, and in this method we rename some
fields for usability and remove extraneous fields.
"""
portfolio = self.__core_dict()
# rename:
# ending_cash -> cash
# period_open -> backtest_start
#
# remove:
# period_close, starting_value,
# cumulative_capital_used, max_leverage, max_capital_used
portfolio['cash'] = portfolio['ending_cash']
portfolio['start_date'] = portfolio['period_open']
portfolio['positions_value'] = portfolio['ending_value']
del (portfolio['ending_cash'])
del (portfolio['period_open'])
del (portfolio['period_close'])
del (portfolio['starting_value'])
del (portfolio['ending_value'])
del (portfolio['cumulative_capital_used'])
del (portfolio['max_leverage'])
del (portfolio['max_capital_used'])
portfolio['positions'] = self.get_positions()
return ndict(portfolio)
def create_txn(sid, price, amount, datetime):
txn = ndict({
'sid': sid,
'amount': amount,
'dt': datetime,
'price': price,
})
return txn
def create_txn(sid, price, amount, datetime):
txn = ndict({
'sid': sid,
'amount': amount,
'dt': datetime,
'price': price,
})
return txn
def test_tracker(self):
trade_count = 100
sid = 133
price = 10.1
price_list = [price] * trade_count
volume = [100] * trade_count
trade_time_increment = datetime.timedelta(days=1)
trade_history = factory.create_trade_history(sid, price_list, volume,
trade_time_increment,
self.trading_environment)
sid2 = 134
price2 = 12.12
price2_list = [price2] * trade_count
trade_history2 = factory.create_trade_history(sid2, price2_list,
volume,
trade_time_increment,
self.trading_environment)
trade_history.extend(trade_history2)
self.trading_environment.period_start = trade_history[0].dt
self.trading_environment.period_end = trade_history[-1].dt
self.trading_environment.capital_base = 1000.0
self.trading_environment.frame_index = [
'sid', 'volume', 'dt', 'price', 'changed'
]
perf_tracker = perf.PerformanceTracker(self.trading_environment)
for event in trade_history:
#create a transaction for all but
#first trade in each sid, to simulate None transaction
if (event.dt != self.trading_environment.period_start):
txn = ndict({
'sid': event.sid,
'amount': -25,
'dt': event.dt,
'price': 10.0,
'commission': 0.50
})
else:
txn = None
event['TRANSACTION'] = txn
perf_tracker.process_event(event)
#we skip two trades, to test case of None transaction
txn_count = len(trade_history) - 2
self.assertEqual(perf_tracker.txn_count, txn_count)
cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
expected_size = txn_count / 2 * -25
self.assertEqual(cumulative_pos.amount, expected_size)
self.assertEqual(
perf_tracker.period_end.replace(hour=0, minute=0, second=0),
perf_tracker.cumulative_risk_metrics.end_date)
def create_transaction(sid, amount, price, dt):
txn = {'sid': sid,
'amount': int(amount),
'dt': dt,
'price': price,
}
transaction = ndict(txn)
return transaction
def create_transaction(sid, amount, price, dt):
txn = {
'sid': sid,
'amount': int(amount),
'dt': dt,
'price': price,
}
transaction = ndict(txn)
return transaction
def test_ndict_deepcopy():
def assert_correctly_copied(orig, copy):
assert nd == nd_dc, \
"Deepcopied ndict should have same keys and values."
nd_dc.z = 3
assert 'z' not in nd, "'z' also added to original ndict."
nd_dc.y = 10
assert nd_dc.y == 10, "value of copied ndict not correctly set."
assert nd.y != 10, "value also set of original ndict."
nd = ndict({'x': 1, 'y': 2})
nd_dc = deepcopy(nd)
assert_correctly_copied(nd, nd_dc)
nd = ndict({'x': [1, 2, 3], 'y': {1: 1}})
nd_dc = deepcopy(nd)
assert_correctly_copied(nd, nd_dc)
nd_dc.x.append(4)
assert nd_dc.x[-1] == 4, "not correctly appended to copied."
assert nd.x[-1] != 4, "also copied to original."
def test_ndict_deepcopy():
def assert_correctly_copied(orig, copy):
assert nd == nd_dc, \
"Deepcopied ndict should have same keys and values."
nd_dc.z = 3
assert 'z' not in nd, "'z' also added to original ndict."
nd_dc.y = 10
assert nd_dc.y == 10, "value of copied ndict not correctly set."
assert nd.y != 10, "value also set of original ndict."
nd = ndict({'x': 1, 'y': 2})
nd_dc = deepcopy(nd)
assert_correctly_copied(nd, nd_dc)
nd = ndict({'x': [1, 2, 3],
'y': {1: 1}})
nd_dc = deepcopy(nd)
assert_correctly_copied(nd, nd_dc)
nd_dc.x.append(4)
assert nd_dc.x[-1] == 4, "not correctly appended to copied."
assert nd.x[-1] != 4, "also copied to original."
def event_with_txn(self, event, no_txn_dt):
#create a transaction for all but
#first trade in each sid, to simulate None transaction
if event.dt != no_txn_dt:
txn = ndict({
'sid': event.sid,
'amount': -25,
'dt': event.dt,
'price': 10.0,
'commission': 0.50
})
else:
txn = None
event['TRANSACTION'] = txn
return event
def event_with_txn(self, event, no_txn_dt):
#create a transaction for all but
#first trade in each sid, to simulate None transaction
if event.dt != no_txn_dt:
txn = ndict({
'sid': event.sid,
'amount': -25,
'dt': event.dt,
'price': 10.0,
'commission': 0.50
})
else:
txn = None
event['TRANSACTION'] = txn
return event
def mapped_data(self):
for row in self.raw_data:
yield ndict(self.apply_mapping(row))
def __missing__(self, key):
pos = Position(key)
self[key] = ndict(pos.to_dict())
return pos
def test_tracker(self, start_dt):
trade_count = 100
sid = 133
price = 10.1
price_list = [price] * trade_count
volume = [100] * trade_count
trade_time_increment = datetime.timedelta(days=1)
trading_environment, start_dt, end_dt = self.create_env(start_dt)
trade_history = factory.create_trade_history(
sid,
price_list,
volume,
trade_time_increment,
trading_environment,
source_id="factory1"
)
sid2 = 134
price2 = 12.12
price2_list = [price2] * trade_count
trade_history2 = factory.create_trade_history(
sid2,
price2_list,
volume,
trade_time_increment,
trading_environment,
source_id="factory2"
)
trade_history.extend(trade_history2)
trading_environment.period_start = trade_history[0].dt
trading_environment.period_end = trade_history[-1].dt
trading_environment.first_open = \
trading_environment.calculate_first_open()
trading_environment.last_close = \
trading_environment.calculate_last_close()
trading_environment.capital_base = 1000.0
trading_environment.frame_index = [
'sid',
'volume',
'dt',
'price',
'changed']
perf_tracker = perf.PerformanceTracker(
trading_environment
)
# date_sort requires 'DONE' messages from each source
events = itertools.chain(trade_history,
[ndict({
'source_id': 'factory1',
'dt': 'DONE',
'type': DATASOURCE_TYPE.TRADE
}),
ndict({
'source_id': 'factory2',
'dt': 'DONE',
'type': DATASOURCE_TYPE.TRADE
})])
events = date_sort(events, ('factory1', 'factory2'))
events = itertools.chain(events,
[ndict({'dt': 'DONE'})])
events = [self.event_with_txn(event, trading_environment)
for event in events]
list(perf_tracker.transform(
itertools.groupby(events, attrgetter('dt'))))
#we skip two trades, to test case of None transaction
txn_count = len(trade_history) - 2
self.assertEqual(perf_tracker.txn_count, txn_count)
cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
expected_size = txn_count / 2 * -25
self.assertEqual(cumulative_pos.amount, expected_size)
self.assertEqual(perf_tracker.last_close,
perf_tracker.cumulative_risk_metrics.end_date)
def test_tracker(self, parameter_comment, days_to_delete):
"""
@days_to_delete - configures which days in the data set we should
remove, used for ensuring that we still return performance messages
even when there is no data.
"""
# This date range covers Columbus day,
# however Columbus day is not a market holiday
#
# October 2008
# Su Mo Tu We Th Fr Sa
# 1 2 3 4
# 5 6 7 8 9 10 11
# 12 13 14 15 16 17 18
# 19 20 21 22 23 24 25
# 26 27 28 29 30 31
start_dt = datetime.datetime(year=2008,
month=10,
day=9,
tzinfo=pytz.utc)
end_dt = datetime.datetime(year=2008,
month=10,
day=16,
tzinfo=pytz.utc)
trade_count = 6
sid = 133
price = 10.1
price_list = [price] * trade_count
volume = [100] * trade_count
trade_time_increment = datetime.timedelta(days=1)
benchmark_returns, treasury_curves = \
factory.load_market_data()
trading_environment = TradingEnvironment(
benchmark_returns,
treasury_curves,
period_start=start_dt,
period_end=end_dt
)
trade_history = factory.create_trade_history(
sid,
price_list,
volume,
trade_time_increment,
trading_environment,
source_id="factory1"
)
sid2 = 134
price2 = 12.12
price2_list = [price2] * trade_count
trade_history2 = factory.create_trade_history(
sid2,
price2_list,
volume,
trade_time_increment,
trading_environment,
source_id="factory2"
)
# 'middle' start of 3 depends on number of days == 7
middle = 3
# First delete from middle
if days_to_delete.middle:
del trade_history[middle:(middle + days_to_delete.middle)]
del trade_history2[middle:(middle + days_to_delete.middle)]
# Delete start
if days_to_delete.start:
del trade_history[:days_to_delete.start]
del trade_history2[:days_to_delete.start]
# Delete from end
if days_to_delete.end:
del trade_history[-days_to_delete.end:]
del trade_history2[-days_to_delete.end:]
trade_history.extend(trade_history2)
trading_environment.first_open = \
trading_environment.calculate_first_open()
trading_environment.last_close = \
trading_environment.calculate_last_close()
trading_environment.capital_base = 1000.0
trading_environment.frame_index = [
'sid',
'volume',
'dt',
'price',
'changed']
perf_tracker = perf.PerformanceTracker(
trading_environment
)
# date_sort requires 'DONE' messages from each source
events = itertools.chain(trade_history,
[ndict({
'source_id': 'factory1',
'dt': 'DONE',
'type': DATASOURCE_TYPE.TRADE
}),
ndict({
'source_id': 'factory2',
'dt': 'DONE',
'type': DATASOURCE_TYPE.TRADE
})])
events = date_sort(events, ('factory1', 'factory2'))
events = itertools.chain(events,
[ndict({'dt': 'DONE'})])
events = [self.event_with_txn(event, trade_history[0].dt)
for event in events]
perf_messages = \
[msg for date, snapshot in
perf_tracker.transform(
itertools.groupby(events, attrgetter('dt')))
for event in snapshot
for msg in event.perf_messages]
#we skip two trades, to test case of None transaction
txn_count = len(trade_history) - 2
self.assertEqual(perf_tracker.txn_count, txn_count)
cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
expected_size = txn_count / 2 * -25
self.assertEqual(cumulative_pos.amount, expected_size)
self.assertEqual(perf_tracker.last_close,
perf_tracker.cumulative_risk_metrics.end_date)
self.assertEqual(len(perf_messages),
trading_environment.days_in_period)
def transaction_sim(self, **params):
""" This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history"""
trade_count = params['trade_count']
trade_interval = params['trade_interval']
trade_delay = params.get('trade_delay')
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
sid = 1
sim_params = factory.create_simulation_parameters()
trade_sim = TransactionSimulator()
price = [10.1] * trade_count
volume = [100] * trade_count
start_date = sim_params.first_open
generated_trades = factory.create_trade_history(
sid,
price,
volume,
trade_interval,
sim_params
)
if alternate:
alternator = -1
else:
alternator = 1
order_date = start_date
for i in xrange(order_count):
order = ndict({
'sid': sid,
'amount': order_amount * alternator ** i,
'dt': order_date
})
trade_sim.place_order(order)
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
# there should now be one open order list stored under the sid
oo = trade_sim.open_orders
self.assertEqual(len(oo), 1)
self.assertTrue(sid in oo)
order_list = oo[sid]
self.assertEqual(order_count, len(order_list))
for i in xrange(order_count):
order = order_list[i]
self.assertEqual(order.sid, sid)
self.assertEqual(order.amount, order_amount * alternator ** i)
tracker = PerformanceTracker(sim_params)
# this approximates the loop inside TradingSimulationClient
transactions = []
for trade in generated_trades:
if trade_delay:
trade.dt = trade.dt + trade_delay
trade_sim.update(trade)
if trade.TRANSACTION:
transactions.append(trade.TRANSACTION)
tracker.process_event(trade)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in xrange(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
cumulative_pos = tracker.cumulative_performance.positions[sid]
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should now be empty
oo = trade_sim.open_orders
self.assertTrue(sid in oo)
order_list = oo[sid]
self.assertEqual(0, len(order_list))
def __missing__(self, key):
pos = Position(key)
self[key] = ndict(pos.to_dict())
return pos
def transaction_sim(self, **params):
""" This is a utility method that asserts expected
results for conversion of orders to transactions given a
trade history"""
trade_count = params['trade_count']
trade_interval = params['trade_interval']
trade_delay = params.get('trade_delay')
order_count = params['order_count']
order_amount = params['order_amount']
order_interval = params['order_interval']
expected_txn_count = params['expected_txn_count']
expected_txn_volume = params['expected_txn_volume']
# optional parameters
# ---------------------
# if present, alternate between long and short sales
alternate = params.get('alternate')
# if present, expect transaction amounts to match orders exactly.
complete_fill = params.get('complete_fill')
sid = 1
trading_environment = factory.create_trading_environment()
trade_sim = TransactionSimulator()
price = [10.1] * trade_count
volume = [100] * trade_count
start_date = trading_environment.first_open
generated_trades = factory.create_trade_history(
sid, price, volume, trade_interval, trading_environment)
if alternate:
alternator = -1
else:
alternator = 1
order_date = start_date
for i in xrange(order_count):
order = ndict({
'sid': sid,
'amount': order_amount * alternator**i,
'dt': order_date
})
trade_sim.place_order(order)
order_date = order_date + order_interval
# move after market orders to just after market next
# market open.
if order_date.hour >= 21:
if order_date.minute >= 00:
order_date = order_date + timedelta(days=1)
order_date = order_date.replace(hour=14, minute=30)
# there should now be one open order list stored under the sid
oo = trade_sim.open_orders
self.assertEqual(len(oo), 1)
self.assertTrue(sid in oo)
order_list = oo[sid]
self.assertEqual(order_count, len(order_list))
for i in xrange(order_count):
order = order_list[i]
self.assertEqual(order.sid, sid)
self.assertEqual(order.amount, order_amount * alternator**i)
tracker = PerformanceTracker(trading_environment)
# this approximates the loop inside TradingSimulationClient
transactions = []
for trade in generated_trades:
if trade_delay:
trade.dt = trade.dt + trade_delay
trade_sim.update(trade)
if trade.TRANSACTION:
transactions.append(trade.TRANSACTION)
tracker.process_event(trade)
if complete_fill:
self.assertEqual(len(transactions), len(order_list))
total_volume = 0
for i in xrange(len(transactions)):
txn = transactions[i]
total_volume += txn.amount
if complete_fill:
order = order_list[i]
self.assertEqual(order.amount, txn.amount)
self.assertEqual(total_volume, expected_txn_volume)
self.assertEqual(len(transactions), expected_txn_count)
cumulative_pos = tracker.cumulative_performance.positions[sid]
self.assertEqual(total_volume, cumulative_pos.amount)
# the open orders should now be empty
oo = trade_sim.open_orders
self.assertTrue(sid in oo)
order_list = oo[sid]
self.assertEqual(0, len(order_list))
def test_tracker(self):
trade_count = 100
sid = 133
price = 10.1
price_list = [price] * trade_count
volume = [100] * trade_count
trade_time_increment = datetime.timedelta(days=1)
trade_history = factory.create_trade_history(
sid,
price_list,
volume,
trade_time_increment,
self.trading_environment
)
sid2 = 134
price2 = 12.12
price2_list = [price2] * trade_count
trade_history2 = factory.create_trade_history(
sid2,
price2_list,
volume,
trade_time_increment,
self.trading_environment
)
trade_history.extend(trade_history2)
self.trading_environment.period_start = trade_history[0].dt
self.trading_environment.period_end = trade_history[-1].dt
self.trading_environment.capital_base = 1000.0
self.trading_environment.frame_index = [
'sid',
'volume',
'dt',
'price',
'changed']
perf_tracker = perf.PerformanceTracker(
self.trading_environment
)
for event in trade_history:
#create a transaction for all but
#first trade in each sid, to simulate None transaction
if(event.dt != self.trading_environment.period_start):
txn = ndict({
'sid': event.sid,
'amount': -25,
'dt': event.dt,
'price': 10.0,
'commission': 0.50
})
else:
txn = None
event['TRANSACTION'] = txn
perf_tracker.process_event(event)
#we skip two trades, to test case of None transaction
txn_count = len(trade_history) - 2
self.assertEqual(perf_tracker.txn_count, txn_count)
cumulative_pos = perf_tracker.cumulative_performance.positions[sid]
expected_size = txn_count / 2 * -25
self.assertEqual(cumulative_pos.amount, expected_size)
self.assertEqual(perf_tracker.period_end.
replace(hour=0, minute=0, second=0),
perf_tracker.cumulative_risk_metrics.end_date)