def test_bars(length=50):
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strat_name = 'test_%d' % length
strategy = MStrategy(strat_name, strategy_params={'length': length})
strategy.bar_interval = 0
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
porto_name = 'portfolio_%d' % length
portfolio = Portfolio(porto_name, None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
log.info("START JOB %s = %s" % (porto_name, datetime.datetime.now()))
simData = DataFeedBars('20100315.SPY.1m.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
# portfolio.stats(write_data=True)
filename = 'TEST_BAR_X_%d.xls' % length
port_stats = portfolio.stats(filename=filename)
print 'portfolio stats'
pprint(port_stats)
log.info("END JOB %s = %s" % (porto_name, datetime.datetime.now()))
return portfolio
def test_multiple_symbols():
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strategy = MStrategy('test_strategy', strategy_params={'length': 10})
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
portfolio = Portfolio('test_porto', None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
log.info("START JOB= %s" % datetime.datetime.now())
## combined file of SPY, IWM, and QQQQ
simData = DataFeedIntraday('20100315.XXX.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
log.info("STAT JOB= %s" % datetime.datetime.now())
port_stats = portfolio.stats(filename='TESTER.xls')
pprint(port_stats)
log.info("END JOB= %s" % datetime.datetime.now())
log.info("LEN DATA JOB= %s" % simData.count)
def test_retrace_strategy(strategy_params, run_id):
latch = DataLatch(3)
s1_order_q = DQueue()
s1_fill_q = DQueue()
strat_name = 'test_%04d' % run_id
## NOTE strategy_params is a dict that the strategy
## uses to initialize itself.
strategy = RetraceStrategy(strat_name, strategy_params=strategy_params)
strategy.bar_interval = 1
strategy.IN_fills = s1_fill_q
strategy.OUT_orders = s1_order_q
strategy.latch = latch
porto_name = 'retrace_%d' % run_id
portfolio = Portfolio(porto_name, None)
portfolio.latch = latch
portfolio.add(strategy)
exchange = Exchange()
exchange.IN_orders = portfolio.OUT_orders
exchange.OUT_fills = portfolio.IN_fills
exchange.latch = latch
exchange.start()
portfolio.start()
strategy.start()
simData = DataFeedDaily('daily.SPY.csv')
for market_data in simData:
latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
strategy.on_data(market_data)
portfolio.on_data(market_data)
exchange.on_data(market_data)
## do any final processing
#strategy.flush()
exchange.shutdown()
portfolio.shutdown()
strategy.shutdown()
exchange.join()
portfolio.join()
strategy.join()
return portfolio.stats()
path = '../derivedData/Monthly'
filenames = [
str(i) + '_' + str(j) + '.csv' for i in [2014, 2015, 2016]
for j in range(1, 13)
]
filenames = filenames[:-4]
for filename in filenames:
predicts = pd.read_csv(os.path.join(path, filename), index_col=0)
us = str.find(filename, '_')
yr = int(filename[:us])
mth = int(filename[us + 1:str.find(filename, '.')])
d = dt.datetime(yr, mth, 1)
budget = min(max_per_month, max_funding + port_rob.spent(d))
for loan_id in predicts.index:
port_rob.add(loan_id, predicts.loc[loan_id, 'weights'] * budget)
univ = accept[accept['issue_d'] == accept.loc[loan_id, 'issue_d']]
port_rand.add(np.random.choice(univ.index, 1)[0], budget / 100)
# Generate results
fv_date = max(port_rob.receipts.keys())
res_sel = port_rob.get_cashflows(fv_date)
res_rand = port_rand.get_cashflows(fv_date)
results = res_sel.join(res_rand, how='outer', lsuffix='_sel', rsuffix='_rand')
results['PVs_sel'] = results['PVs_sel'] / results['n_loans_sel']
results['PVs_rand'] = results['PVs_rand'] / results['n_loans_rand']
offered = accept[['issue_d', 'funded_amnt']].copy()
offered['issue_d'] = offered['issue_d'].apply(
lambda i: dt.datetime.strptime(str(i), '%Y%m%d'))
offered = offered.groupby('issue_d').agg(['count', 'sum'])
offered.columns = offered.columns.get_level_values(1)
boost.fit(X_train_s, y_train)
print(f'IS R^2: {boost.score(X_train_s, y_train):.2%}')
# Determine what we have to invest this month
if str(date)[4:6] == '01':
print(f'Simulating for {date}')
d = dt.datetime.strptime(str(date), '%Y%m%d')
budget = min(max_per_month, max_funding + port_sel.spent(d))
n_to_buy = int(budget / max_per_loan)
# Select the loans that will issue this month
bools = X['issue_d'] == date
idx = X.index[bools]
# Predict PVs for them
rets = pd.Series(boost.predict(X_s[bools, :]), index=idx).sort_values()
# Purchase the selected and random loans
bought = len([
port_sel.add(rets.index[-i], max_per_loan)
for i in range(1, n_to_buy + 1) if rets.iloc[-i] >= min_ret
])
[
port_rand.add(ix, max_per_loan)
for ix in np.random.choice(idx, bought, replace=False)
]
# Generate results
fv_date = max(port_sel.receipts.keys())
res_sel = port_sel.get_cashflows(fv_date)
res_rand = port_rand.get_cashflows(fv_date)
results = res_sel.join(res_rand, how='outer', lsuffix='_sel', rsuffix='_rand')
results['PVs_sel'] = results['PVs_sel'] / results['n_loans_sel']
results['PVs_rand'] = results['PVs_rand'] / results['n_loans_rand']
offered = accept[['issue_d', 'funded_amnt']].copy()
class Simulator(object):
def __init__(self):
self.latch = None
self.strategies = []
self.portfolio = Portfolio('portfolio', None)
self.exchange = Exchange()
self.portfolio.IN_fills = self.exchange.OUT_fills
self.stats = None
self.verbose = True
## call on_EOD funcs at end of data file
self.reset_on_EOD = True
self.scoring_function = None
def add_strategy(self, strategy):
self.strategies.append(strategy)
strategy.IN_fills = DQueue()
self.portfolio.add(strategy)
self.exchange.add(strategy)
def run(self, datafeed):
log.info("reset_on_EOD = %s" % self.reset_on_EOD)
self.latch = DataLatch(len(self.strategies) + 2)
self.portfolio.latch = self.latch
self.exchange.latch = self.latch
for s in self.strategies:
s.latch = self.latch
bg = datetime.datetime.now()
if self.verbose: log.info('Sim Start: %s' % bg)
for market_data in datafeed:
if market_data != DataFeed.SENTINEL:
self.latch.trap(market_data)
## ORDER MATTERS!
## this allows submit-fill loop to happen in a single on_data() event
for s in self.strategies:
s.on_data_sim(market_data)
self.exchange.on_data_sim(market_data)
self.portfolio.on_data_sim(market_data)
else:
if self.reset_on_EOD:
## handle EOD processing
self.portfolio.on_EOD()
for s in self.strategies:
s.on_EOD()
self.exchange.on_EOD()
nd = datetime.datetime.now()
if self.verbose:
log.info('Sim Completed: %s' % nd)
log.info('Time Elapsed: %s' % (nd - bg))
return self.dump()
def dump(self):
if not self.scoring_function:
self.scoring_function = fitness_function
if not self.stats:
self.stats = self.portfolio.stats()
self.stats['_score'] = self.scoring_function(self.stats)
summary = OrderedDict()
header = [
'_score', 'cnt', 'w_pct', 'pr', 'pnl', 'mtm_pnl', 'max_equ',
'max_dd'
]
for k in header:
summary[k] = self.stats[k]
table = PrettyTable(header)
table.add_row(summary.values())
header.pop(1) ## remove 'cnt' label
for k in header:
table.float_format[k] = '0.2'
if self.verbose: log.info('\n%s' % table)
## return stat summary for potential use elsewhere
#return summary
return self.stats
## show the equity curve
def show(self):
if not self.stats:
self.stats = self.portfolio.stats()
curve_df = self.portfolio.storage['curve']['portfolio']
curve_df.plot(x='timestamp')
## write output
def write(self, filename):
root = ".".join(filename.split('.')[:-1])
if not root: root = filename
xls = ".".join([root, 'xls'])
pkl = ".".join([root, 'pkl'])
log.info('writing pickle file: %s' % pkl)
self.portfolio.write(filename=pkl)
self.portfolio.to_excel(filename=xls)
