def get_historical_fundamentals(ticker: str, d: date,
manager: CollectionManager):
"""
Gets all of the fundamental data for a ticker before some date and after
:param ticker: stock ticker
:param d: date
:param manager: collection manager for the fundamentals database
:return: past fundamentals, announcement dates, and future (test) fundamentals
"""
current_day = dt(d.year, d.month, d.day)
allTickersFundamentals = manager.find({
'ticker': ticker,
'date': {
'$lte': current_day
}
}).sort_values('date')
test = manager.find({
'ticker': ticker,
'date': {
'$gte': current_day
}
}).sort_values('date')
return allTickersFundamentals[features], [
announce.date()
for announce in allTickersFundamentals['date'].tolist()
], test
def __init__(self,
start,
capital,
model,
loss,
p=0.015,
sharePer=0.5,
stop='2018-02-05'):
self.manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
self.dates = self.manager.dates()
self.startIndex = self.dates.index(start) + 1
self.stop = stop
self.stopIndex = self.dates.index(stop)
self.suggestor = SectorSuggestor(self.startIndex)
self.suggestor.build_sector_NN()
self.sectorModels = {}
self.portfolio = {}
self.loss = loss
self.model = model
self.p = p
self.sharePer = sharePer
self.startDate = date
self.startingCapital = capital
def calculate_performance(ticker, dates1: list, dates2: list):
"""
Gets how much the stock has changed since the last
quarter reportings.
:param ticker: stock ticker
:param date1: beginining of the quarter
:param date2: end of the quarter
:return: percent change in stock price
"""
ticker = ticker.lower()
manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
prices = manager.find({'ticker': ticker})
dates = [
dt.strptime(priceDate, '%Y-%m-%d').date()
for priceDate in prices['date']
]
pricesStart = [
prices[prices['date'] == str(d1)]['vwap'].values[0]
for d1 in get_next_trading_day(dates, dates1)
]
pricesEnd = [
prices[prices['date'] == str(d2)]['vwap'].values[0]
for d2 in get_next_trading_day(dates, dates2)
]
manager.close()
performances = [((p[0] - p[1]) / p[0])
for p in zip(pricesStart, pricesEnd)]
return performances
def __init__(self, manager: CollectionManager, agent: InvestorAgent,
startDay: int):
self.timeperiod = manager.dates()
self.manager = manager
self.agent = agent
self.day = startDay
self.currentDate = self.timeperiod[self.day]
def async_grid(args):
"""
Runs grid search to see parameter values
:param args: args for the trade method
:return: None
"""
manager = CollectionManager('grid_search', 'AlgoTradingDB')
try:
stock = args[-1]
result = trade(*args)
log(f"{args} SUCCESS!\n{result}\n")
save_results(result, manager, stock)
except Exception as e:
log(f"{args} failed {e}")
manager.close()
def get_data_by_sector(stocks, sector):
"""
Gets price data from the database for a given sector
:param stocks: list of stocks
:param sector: given sector
:return: writes to CSV
"""
manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
dates = manager.dates()
data = pd.DataFrame()
for stock in stocks:
stocksData = avg_price_timeseries(manager, stock.lower(), dates)
if len(stocksData) != 1259:
continue
data[stock] = stocksData
data.to_csv('sectorAnalysis/{0}.csv'.format(sector))
manager.close()
def run_simulation(self):
"""
Runs the full trading framework.
:return: None
"""
warnings.filterwarnings("ignore")
sectors = []
for day in range(self.startIndex, self.stopIndex):
print(f'--------DAY: {day - self.startIndex +1}--------')
# Suggest a Sector
sectorIDtoInvestIn = self.suggestor.predict_sector(day)
sectorToInvestIn = id_to_sector[sectorIDtoInvestIn]
print(f'I suggest investing in the {sectorToInvestIn} sector')
if len(sectors) != 0:
if sectors[-1] == sectorToInvestIn:
print('Already holding positions in this sector...')
sectors.append(sectorToInvestIn)
continue
sectors.append(sectorToInvestIn)
# Suggest top three stocks in this market
if sectorToInvestIn not in self.sectorModels:
stockModel = StockSuggestor(sectorToInvestIn, day,
self.dates[day])
self.sectorModels[sectorToInvestIn] = stockModel
stockModel.build_network()
else:
stockModel = self.sectorModels[sectorToInvestIn]
stockToInvestIn, untilThisDate = stockModel.predict_stock(
self.dates[day])
print(
f'I suggest investing in the following stock: {stockToInvestIn}'
)
self.trade_stock(stockToInvestIn.lower(), self.startDate,
self.dates[day + 50])
print(f'SECTORS INVESTED IN: {sectors}')
resultManager = CollectionManager('trading_results', 'AlgoTradingDB')
resultManager.insert(self.portfolio)
def fit_and_plot(self, series, dates, window=0):
"""
Fits the ARIMA model and plots the prediction results
:param series: time series
:param dates: dates for fitting
:param window: sliding window
:return: None
"""
manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
size = len(dates) - 518
train, test = series[0:size], series[size:len(series)]
predictions = list()
w = 0
for t in range(len(test) + 1):
model = ARIMA(train, order=(self.p, self.d, self.q))
model_fit = model.fit(disp=0)
output = model_fit.forecast()
yhat = output[0][0]
predictions.append(yhat)
obs = yhat
if w == window and t != len(test):
train.append(test[t])
w = 0
else:
train.append(obs)
w += 1
error = mean_squared_error(test, predictions[:len(test)])
print(error)
actual, days = create_timeseries(manager, self.ticker)
days = [days[x] for x in range(0, len(days), 2)]
actual = [actual[y] for y in range(0, len(actual), 2)]
predictions = [predictions[p] for p in range(0, len(predictions), 2)]
plt.plot(days, actual, color='black', label='Actual')
plt.plot(days[1000:],
predictions[:-1],
color='red',
label='LSTM predictions')
plt.xlabel('day')
plt.title(self.ticker)
plt.ylabel('price')
plt.legend(loc=2)
plt.savefig('plots/ARIMA/ARIMA_{0}_predictions.pdf'.format(
self.ticker))
plt.show()
def add_fundamentals_to_db():
"""
Adds the fundamental data to the database from a json file
:return:None
"""
fundFile = 'sectorAnalysis/fundamentals/combinedFundamentals.json'
funds = pd.read_json(fundFile)
manager = CollectionManager('10y_Fundamentals', 'AlgoTradingDB')
for index, row in funds.iterrows():
document = row.to_dict()
manager.insert(document, is_dictionary=True)
manager.close()
from DataHandler.putAndGetData import get_stock_data,add_data
from DataHandler.mongoObjects import CollectionManager,MongoDocument
from pymongo import MongoClient
import DataHandler.apiCall as api
if __name__ == '__main__':
manager = CollectionManager('daily_technicals', 'AlgoTradingDB')
add_data(manager,function=api.iex_1d,unwantedFields=['label','average','changeOverTime',
'marketAverage','marketChangeOverTime'])
features = list(dataset.columns.values)
for feature in features:
series = dataset[feature]
diff = list()
for i in range(interval, len(series)):
value = series[i] - series[i - interval]
diff.append(value)
differenced[feature] = diff
return differenced['price']
startDay = 1153
stopDay = 1258
stocks = ['googl','nvda','vz','wmt']
model = 'LSTM'
manager = CollectionManager('5Y_technicals','AlgoTradingDB')
dates = manager.dates()
actualPrice = avg_price_timeseries(manager, 'spy', dates[startDay:stopDay])
diff_price = pd.DataFrame()
diff_price['price'] = actualPrice
diff_price = list(diff(diff_price))
portfolioTrades = f'/Users/adelekap/Documents/capstone_algo_trading/comparison/{model}/trades/'
capitalDates = []
all = pd.DataFrame()
for stock in stocks:
stockData = pd.read_csv(portfolioTrades+stock+'.csv').iloc[:(stopDay-startDay-4),:]
all[stock] = stockData['CurrentCapital']
capitalDates = stockData['Date'].unique()
def trade(loss,
statsModel,
p,
sharePer,
startDate,
startingCapital,
stop,
ticker,
epochs=1,
neurons=1,
plotting=False):
"""
TRADES a given stock for a period of time
:param loss: amount of loss willing to take
:param statsModel: string of the predictive model to be used
:param p: p value for the strategy
:param sharePer: share percentage for the strategy
:param startDate: date to start YYYY-MM-DD
:param startingCapital: capital to start trading with
:param stop: stop date YYYY-MM-DD
:param ticker: stock ticker
:param plotting: whether you want to plot or not
:return: results (dictionary)
"""
logger = Logger('comparison/{0}/trades/{1}.csv'.format(statsModel, ticker))
logger.log(
'Date,Open/Close,PositionNum,Type,Price,Shares,Investment,Profit,CurrentCapital'
)
positionOpenNum = 0
positionCloseNum = 0
warnings.filterwarnings("ignore")
"""Initialize Environment"""
# Data
manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
# Time
dates = manager.dates()
currentDate = startDate
startDay = dates.index(currentDate)
stopDay = dates.index(stop)
# bar = utils.ProgressBar(stopDay - startDay)
results = {'p': p, 'sharePer': sharePer}
# Predictive Model
if statsModel == 'Arima':
model = ArimaModel(1, 1, 0, ticker)
k = 5
if statsModel == 'LSTM':
model = NeuralNet(ticker, manager, startDay)
model.create_network()
model.train_network()
k = 1
if statsModel == 'SVM':
model = SVM(100, 0.01, ticker, manager, startDay)
model.fit()
k = 5
# Investor, Strategy and Trading Environment
stopLoss = (1 - loss) * startingCapital
tradingStrategy = Strategy(model, manager, ticker, currentDate, stopLoss,
p)
investor = InvestorAgent(startingCapital, tradingStrategy, startDay)
environment = Environment(manager, investor, startDay)
# Simulate Trading Environment
print(f'Starting Trading {ticker}')
for d in range(startDay, stopDay):
if len(investor.positions):
for position in investor.positions:
currentPrice = investor.check_price(environment.currentDate)
actionDay = utils.laterDate(position.startDate,
position.holdTime)
if environment.currentDate == actionDay or position.at_trigger_point(
currentPrice):
sellMessage = position.sell(investor, currentPrice)
logger.log(
f'{environment.currentDate},close,{position.id},' +
sellMessage)
positionCloseNum += 1
T = investor.strategy.arithmetic_returns(k, environment.day)
sig = investor.signal(T)
if sig != 0:
message = investor.strategy.make_position(investor,
sig,
environment.currentDate,
stopLoss,
sharePer,
posNum=positionOpenNum)
if message != None:
logger.log(
f'{environment.currentDate},open,{positionOpenNum},' +
message)
positionOpenNum += 1
environment.update_total_assets(investor)
if d != stopDay - 1:
environment.increment_day(investor.strategy)
# bar.progress()
"""PLOTTING"""
print('PLOTTING')
actualPrice = avg_price_timeseries(manager, ticker,
dates[startDay:stopDay])
if not len(investor.capitalHistory):
expReturn = 0
else:
expReturn = round(
((investor.totalAssetHistory[len(investor.totalAssetHistory) - 1] -
startingCapital) / startingCapital) * 100)
gain = str(expReturn) + '%'
possible = round(
((actualPrice[-1] - actualPrice[0]) / actualPrice[0]) * 100, 1)
mdd = utils.MDD(investor.totalAssetHistory)
if plotting:
utils.plot_capital(investor.totalAssetHistory,
dates[startDay:stopDay],
ticker,
actualPrice,
gain,
mdd,
possible,
model=statsModel)
results['MDD'] = mdd
results['return'] = expReturn
results['possible'] = possible
etf = avg_price_timeseries(manager, 'spy', dates[startDay:stopDay])
print(f'RETURN ON INVESTMENT: {expReturn}')
print(f'Number of Positions Opened: {positionOpenNum}')
print(f'Number of Positions Closed:{positionCloseNum}')
print(
f'Sharpe Ratio = {sharpe_ratio(investor.totalAssetHistory, etf, possible)}'
)
manager.close()
return (results)
class TradingFramework():
def __init__(self,
start,
capital,
model,
loss,
p=0.015,
sharePer=0.5,
stop='2018-02-05'):
self.manager = CollectionManager('5Y_technicals', 'AlgoTradingDB')
self.dates = self.manager.dates()
self.startIndex = self.dates.index(start) + 1
self.stop = stop
self.stopIndex = self.dates.index(stop)
self.suggestor = SectorSuggestor(self.startIndex)
self.suggestor.build_sector_NN()
self.sectorModels = {}
self.portfolio = {}
self.loss = loss
self.model = model
self.p = p
self.sharePer = sharePer
self.startDate = date
self.startingCapital = capital
def run_simulation(self):
"""
Runs the full trading framework.
:return: None
"""
warnings.filterwarnings("ignore")
sectors = []
for day in range(self.startIndex, self.stopIndex):
print(f'--------DAY: {day - self.startIndex +1}--------')
# Suggest a Sector
sectorIDtoInvestIn = self.suggestor.predict_sector(day)
sectorToInvestIn = id_to_sector[sectorIDtoInvestIn]
print(f'I suggest investing in the {sectorToInvestIn} sector')
if len(sectors) != 0:
if sectors[-1] == sectorToInvestIn:
print('Already holding positions in this sector...')
sectors.append(sectorToInvestIn)
continue
sectors.append(sectorToInvestIn)
# Suggest top three stocks in this market
if sectorToInvestIn not in self.sectorModels:
stockModel = StockSuggestor(sectorToInvestIn, day,
self.dates[day])
self.sectorModels[sectorToInvestIn] = stockModel
stockModel.build_network()
else:
stockModel = self.sectorModels[sectorToInvestIn]
stockToInvestIn, untilThisDate = stockModel.predict_stock(
self.dates[day])
print(
f'I suggest investing in the following stock: {stockToInvestIn}'
)
self.trade_stock(stockToInvestIn.lower(), self.startDate,
self.dates[day + 50])
print(f'SECTORS INVESTED IN: {sectors}')
resultManager = CollectionManager('trading_results', 'AlgoTradingDB')
resultManager.insert(self.portfolio)
def trade_stock(self, ticker, start, stop):
self.portfolio[ticker] = (trade(self.loss,
self.model,
self.p,
self.sharePer,
start,
self.startingCapital,
stop,
ticker,
plotting=True))
def get_all_fundamentals(stocks: list, tradeDate: date):
"""
Gets all of the fundamentals for a list of tickers and list of quarters
:param tickers: stocks
:param quarters: list of quarters
:param final: whether this is a final prediction
:return: Xs and ys
"""
manager = CollectionManager('10y_Fundamentals', 'AlgoTradingDB')
tickers_set = set(stocks)
all_fundamental_tickers = set(manager.find({})["ticker"])
tickers = list(tickers_set.intersection(all_fundamental_tickers))
allFundamentals = pd.DataFrame()
performances = pd.DataFrame()
quarters = 17
allTest = pd.DataFrame()
testDates = []
for ticker in tickers:
data, announcementDates, test = get_historical_fundamentals(
ticker, tradeDate, manager)
nextAnnouncementDates = announcementDates[1:] + [
dt.strptime('2018-02-05', '%Y-%m-%d').date()
]
performance = calculate_performance(ticker, announcementDates,
nextAnnouncementDates)
if len(testDates) == 0:
testDates = test['date'].tolist()
if len(performance) != 17:
performance = performance[len(performance) - 17:]
performances[ticker] = performance
else:
performances[ticker] = performance
for index, funds in data.iterrows():
tempDF = pd.DataFrame()
tempDF['fundamentals'] = list(funds)[:-1]
tempDF['ticker'] = [ticker for i in range(len(funds) - 1)]
tempDF['quarter'] = [index for j in range(len(funds) - 1)]
allFundamentals = pd.concat([allFundamentals, tempDF])
for index, testFunds in test.iterrows():
temp = pd.DataFrame()
temp['fundamentals'] = list(testFunds)[:-1]
temp['ticker'] = [ticker for k in range(len(testFunds) - 1)]
temp['quarter'] = [index for l in range(len(testFunds) - 1)]
allTest = pd.concat([allTest, temp])
manager.close()
trainingData = []
for quarter in range(quarters):
q = []
for ticker in tickers:
tickerdata = allFundamentals[allFundamentals['ticker'] == ticker]
quarterdata = tickerdata[tickerdata['quarter'] ==
quarter]['fundamentals']
q.append(quarterdata.tolist())
trainingData.append(np.array(q))
trainingDataX = np.array(trainingData)
trainingDataY = find_best_stock(performances)
allTestY = []
quarterLen = len(allTest['quarter'].unique())
if quarterLen == 1:
fix = allTest.copy()
fix['quarter'] = [2 for i in range(len(fix))]
allTest = pd.concat([allTest, fix])
for testQuarter in range(2):
testQ = []
for tick in tickers:
tickData = allTest[allTest['ticker'] == tick]
testQuarterData = tickData[tickData['quarter'] ==
testQuarter]['fundamentals']
if testQuarterData.shape[0] != 15:
print('ERROR ' + tick)
testQ.append(testQuarterData.tolist()[:-4])
allTestY.append(np.array(testQ))
return trainingDataX, trainingDataY, np.array(allTestY), testDates, tickers
from environment import trade
from utils import save_results
import pandas as pd
from DataHandler.mongoObjects import CollectionManager
if __name__ == '__main__':
stocks = [
symbol.lower() for symbol in list(pd.read_csv('stocks.csv')['Symbol'])
]
manager = CollectionManager('trading_results', 'AlgoTradingDB')
loss = 0.30
startingCapital = 15000
p = 0.025
sharePer = 0.17
startDate = '2018-01-03'
stopDate = '2018-02-05'
models = ['Arima', 'LSTM']
epochs = 10
neurons = 4
for stock in stocks:
for model in models:
result = trade(loss,
model,
p,
sharePer,
startDate,
startingCapital,
stopDate,