def getProfitSpeed(self, args):
'''
PRIVATE METHOD
args is a list passed in the following format: [ticker, date]
Returns a list in the following format: [profitSpeed, ticker]
'''
ticker = args[0]
date = args[1]
try:
dataframe = database.getDataframe(ticker, dateRange=[date,date], dataFields=["Profit Speed"])# .at[date,"Profit Speed"]
#print(dataframe)
try:
isMissing = pd.isnull(dataframe).loc[date,"Profit Speed"]
except Exception as e:
isMissing = True
if (isMissing):
#Profit speed data is missing. Return nothing
del dataframe
pass
else:
profitSpeed = dataframe.loc[date,"Profit Speed"]
del dataframe
return [profitSpeed, ticker]
except:
pass
def selectStocksToSell(self,
listOfOwnedStocks,
date=False,
genricParameters=[]):
'''
Selects which stocks to sell.
Passing a date parameter will select stocks for the given date. Ommitting it will
select stocks based on realtime data.
Returns a list of stocks to sell.
'''
if (not date):
raise ValueError(
"Real time selection not availible for this selector")
else:
tickerList = listOfOwnedStocks
stocksToSell = []
for ticker in tickerList:
try:
stockData = database.getDataframe(
ticker,
dataFields=["2D Discrete Moementum"],
dateRange=[date, date])
twoDayDiscreteMomentum = float(
stockData["2D Discrete Moementum"])
if (twoDayDiscreteMomentum < 1):
stocksToSell.append(ticker)
except:
pass
return stocksToSell
def joeysPaintFilter(args):
'''
Filters a single stock based on Joey's beautiful piecewise functions drawn in paint.
Returns False if the stock did not pass the filter
'''
ticker = args[0]
date = args[1]
dataFields = [
"2 Day Normalized Slope", "5 Day Normalized Slope",
"2D Discrete Moementum", "5D Discrete Moementum"
]
stockData = database.getDataframe(ticker,
dataFields=dataFields,
dateRange=[date, date])
twoDaySlope = float(stockData["2 Day Normalized Slope"])
fiveDaySlope = float(stockData["5 Day Normalized Slope"])
if ((twoDaySlope > 0.2) and (fiveDaySlope > -4.5)
and (fiveDaySlope > ((-1.8333 * twoDaySlope) + 1.3666))):
pass
else:
#Doesn't pass the slope piecewise filter
return False
twoDayDiscreteMomentum = float(stockData["2D Discrete Moementum"])
fiveDayDiscreteMomentum = float(stockData["5D Discrete Moementum"])
if ((fiveDayDiscreteMomentum < 3) and (twoDayDiscreteMomentum > 0)):
pass
else:
return False
return ticker
def addDatapointsFromStock(args):
'''
Returns the datapoints for a single stock, or False if data in unavailable or missing
args is a tuple passed in the following format: (ticker, date)
Returns a tuple in the following format:
(string ticker, array_type datapoints)
'''
ticker = args[0]
date = args[1]
if (not date):
raise ValueError("Real time selection not availible for this selector")
try:
#Get all data for stock
dataFields = [
"2 Day Normalized Slope", "5 Day Normalized Slope",
"2 Day Normalized Momentum", "5 Day Normalized Momentum",
"2D Discrete Moementum", "5D Discrete Moementum",
"Standard Dev Normalized"
]
stockData = database.getDataframe(ticker, dataFields=dataFields)
#Get the input datapoints for this day
daysToPass = 10
endDateIndex = stockData.index.get_loc(date)[0]
startDateIndex = endDateIndex + daysToPass
dataframeSlice = stockData[endDateIndex:startDateIndex]
#Check for missing data
dataframeSlice.dropna(inplace=True)
if (len(dataframeSlice) < daysToPass):
return False
networkInputs = dataframeSlice.values.flatten()
returnTuple = (ticker, networkInputs)
return returnTuple
except Exception as e:
#print(e)
return False
def updateAssets(self, date=False):
'''
Updates the net value of all currently owned stock assets.
If date is not included, the current real-time price is used.
'''
self.stockAssets = 0
for ticker in self.stocksOwned:
quantityOwned = self.stocksOwned.get(ticker)[0]
if (date):
tickerData = database.getDataframe(ticker)
try:
isMissing = pd.isnull(tickerData).loc[date, "Open"].bool()
if (not isMissing):
value = tickerData.loc[date, "Open"]
else:
value = self.stocksOwned.get(ticker)[3]
except:
#Using old value if current value cannot be found
value = self.stocksOwned.get(ticker)[3]
#Check to see if any splits occured
try:
splitData = pd.DataFrame.from_csv("Data/SplitData/" +
ticker +
".csv").values.tolist()
for line in splitData[0]:
splitDict = json.loads(line.replace("'", '"'))
if splitDict['paymentDate'] == date:
#print(splitDict)
self.stocksOwned.get(
ticker)[0] *= splitDict['toFactor']
quantityOwned = self.stocksOwned.get(ticker)[0]
#print("SPLIT:",ticker)
except Exception as e:
pass
else:
value = rsd.getCurrentPrice(ticker)
self.stockAssets += int(value * quantityOwned * 100)
def addDatapointsFromStock(ticker):
DataPoints = []
try:
dataFields = ["2 Day Normalized Slope", "5 Day Normalized Slope", "2 Day Normalized Momentum", "5 Day Normalized Momentum", "2D Discrete Moementum", "5D Discrete Moementum", "Standard Dev Normalized", "Profit Speed"]
stockData = database.getDataframe(ticker, dataFields= dataFields, dateRange=["2010-01-01", "2016-06-06"])
stockData.dropna(inplace = True) #Remove any rows with missing data
#Create input output pairs based on amount of days to include
chunks = []
for i in range(0, len(stockData)-daysToPass, 1):
chunkFrame = stockData.iloc[i:i+daysToPass] #chunk is the dataframe containing all the days to include in this datapoint
chunk = (chunkFrame[dataFields[0:-1]].values.flatten() , float(chunkFrame.iloc[0]["Profit Speed"])) #extract input data and output profitspeed
chunks.append(chunk)
#Seperate dataframe into Positive profit speeds and negative profit speeds
positiveProfitSpeedList = [i for i in chunks if (i[1] > 0)]
negativeProfitSpeedList = [i for i in chunks if (i[1] <= 0)]
#Make sure datapoints are equally distributed between positive and negative profit speeds
numberOfPositives = len(positiveProfitSpeedList)
numberOfNegatives = len(negativeProfitSpeedList)
if (numberOfPositives > numberOfNegatives):
positiveProfitSpeedList = positiveProfitSpeedList[0:numberOfNegatives]
elif (numberOfNegatives > numberOfPositives):
negativeProfitSpeedList = negativeProfitSpeedList[0:numberOfPositives]
DataPoints += positiveProfitSpeedList
DataPoints += negativeProfitSpeedList
return DataPoints
except Exception as e:
#print(e)
return False
def selectStocksToBuy(self,
maxNumberOfStocks,
date=False,
customTickerList=False,
genricParameters=[]):
'''
Selects which stocks to buy, and the proportion of funds to be allocated to each.
maximumNumberOfStocks is the highest number of stocks the caller wants to select.
Passing a date parameter will select stocks for the given date. Ommitting it will
select stocks based on realtime data.
Passing a customTickerList will only analyze tickers included in that list.
Returns a list of in the following format: [ [Ticker1, RatioOfFundAllocation1], [Ticker2, RatioOfFundAllocation2], ... ] .
All ratios will add up to 1.
'''
if (not date):
raise ValueError(
"Real time selection not availible for this selector")
else:
if (customTickerList):
tickerList = customTickerList
else:
tickerList = database.getTickerList()
stocksToConsider = self.filterStocksToConsider(tickerList, date)
if (len(stocksToConsider) > maxNumberOfStocks):
stocksToConsider = stocksToConsider[0:maxNumberOfStocks]
else:
pass
numberOfStocks = len(stocksToConsider)
stocksToBuy = [(i, 1 / numberOfStocks) for i in stocksToConsider]
return stocksToBuy
twoDayDiscreteMomentum = float(stockData["2D Discrete Moementum"])
fiveDayDiscreteMomentum = float(stockData["5D Discrete Moementum"])
if ((fiveDayDiscreteMomentum < 3) and (twoDayDiscreteMomentum > 0)):
pass
else:
return False
return ticker
if __name__ == '__main__':
selector = MehSelector()
date = "2018-06-14"
numberOfStocks = 4000
tickerList = database.getTickerList(randomize=True)[0:numberOfStocks]
#tickerList = ["TSLA"]
selectedStocks = selector.selectStocksToBuy(10, date, tickerList)
print(selectedStocks)
for selection in selectedStocks:
ticker = selection[0]
stockData = database.getDataframe(ticker,
dataFields=[
"Profit Speed",
"2 Day Normalized Momentum",
"5 Day Normalized Momentum",
"2D Discrete Moementum",
"5D Discrete Moementum"
],
dateRange=[date, date])
print(stockData)
def runSimulation(account, dateRange, startingDeposit, selector, sampleSize=False, customTickerList=False, preloadToMemory=False, depositAmount=False, depositFrequency=False, comission=10, PrintToTerminal=True):
'''
Runs a single simulation. Saves results to a csv file.
-Daterange must be a 2 element list, in the following format: [[<start date>], [<end date>]], date format = string "YYYY-MM-DD".
-depositFrequency is how often (in days) to deposit funds into your trading account.
-selector is a StockSelectionInterface object.
-Passing a customTickerList will run the simulation using only the tickers included in the list.
'''
#Check for valid parameters
if ((depositFrequency) and (not depositAmount)):
raise ValueError("Deposit frequency set without deposit amount.")
if ((depositAmount) and (not depositFrequency)):
raise ValueError("Deposit amount set without deposit frequency.")
#Instaniate objects
if (PrintToTerminal):
print("\nGetting tickers...")
if (customTickerList):
tickerList = customTickerList
elif (sampleSize):
tickerList = database.getTickerList(randomize=True, numberOfShuffles=2)[:sampleSize]
else:
tickerList = database.getTickerList()
if (preloadToMemory):
print("Preloading stock data to memory...")
database.loadDatabaseToMemory(tickerList)
#Set starting balance and comission
account.depositFunds(startingDeposit)
account.setCommision(comission)
#Extract daterange
startDate = dateRange[0]
endDate = dateRange[1]
#Progress bar header
if (PrintToTerminal):
print ("\nRuning Simulation...\n")
print ("Selector: " + selector.getName()) #NOTE Don't forget to set your self.name property in you selector constructor
print ("Daterange: "+startDate+" to "+endDate)
print ("-------------------------------------------\n")
sys.stdout.write("\r")
sys.stdout.write("0.0%")
sys.stdout.flush()
daysSinceLastDeposit = 0
#Begin simulation
for date in utils.daterange(startDate, endDate):
#Check if market is open
if (utils.isWeekday(date)):
#Selects which stocks to sell
ownedStocks = account.getOwnedStocks()
ownedTickers = []
for ticker in ownedStocks:
ownedTickers.append(ticker)
stocksToSell = selector.selectStocksToSell(ownedTickers, date=date)
#Sells stocks
account.placeSellOrders(stocksToSell, date)
#Selects which stocks to buy
availibleFunds = account.getBalance()
numberOfStocksToBuy = selector.numberOfStocksToBuy(availibleFunds)
stocksToBuy = selector.selectStocksToBuy(numberOfStocksToBuy, date=date, customTickerList=tickerList)
buyOrders = []
for stock in stocksToBuy:
ticker = stock[0]
price = database.getDataframe(ticker, [date,date], ["Open"]).loc[date, "Open"]
quantity = int((stock[1]*(availibleFunds-(len(stocksToBuy)*comission))) / price)
if quantity>0:
buyOrders.append([ticker, quantity])
#Buys stocks
account.placeBuyOrders(buyOrders, date)
if (depositFrequency):
daysSinceLastDeposit += 1
if (daysSinceLastDeposit == depositFrequency):
account.depositFunds(depositAmount)
daysSinceLastDeposit = 0
#Progress bar
if (PrintToTerminal):
completed = utils.getDayDifference(startDate, date)
totalToDo = utils.getDayDifference(startDate, endDate)
percentage = int(float(completed*1000)/(totalToDo-1))/10.0
sys.stdout.write("\r")
sys.stdout.write(str(percentage)+"%")
sys.stdout.flush()
if (PrintToTerminal):
print("\n")
#Save logs
account.saveHistory(selector.getName())
def placeSellOrders(self, listOfTickers, date, simulation=True):
'''
places a sell order for every ticker in list of tickers.
Will sell ALL stocks owned of that ticker
Date is the current date, in string format "YYYY-MM-DD".
Setting simulation to False will actually execute the sell orders (not yet implimented)
'''
if (not simulation):
pass #actually execute sell orders
action = "Sell"
executedOrders = []
completedBuySellCyles = []
for ticker in listOfTickers:
tickerData = database.getDataframe(ticker)
openPrice = tickerData.loc[date, "Close"].values[0]
if np.isnan(openPrice):
date_index = tickerData.index.get_loc(date)
openPrice = tickerData.iloc[date_index + 1, "Close"]
quantity = self.stocksOwned[ticker][0]
buyPrice = self.stocksOwned[ticker][1]
buyDate = self.stocksOwned[ticker][2]
tradeIncome = int(openPrice * quantity * 100) - self.commision
if (ticker in self.stocksOwned):
#We own the stock
self.balance += tradeIncome #*100 since balance is in cents
del self.stocksOwned[ticker] #Remove stock from stocksOwned
if (simulation):
self.updateAssets(date)
else:
self.updateAssets()
totalAssets = float(self.balance + self.stockAssets) / 100
timeStamp = time.time()
timeOfExecution = datetime.datetime.fromtimestamp(
timeStamp).strftime('%Y-%m-%d %H:%M:%S')
executedOrders.append([
date, ticker, quantity,
float(openPrice), action, timeOfExecution, totalAssets,
float(self.balance) / 100,
float(self.stockAssets) / 100
])
tradeProfit = (
tradeIncome - int(quantity * buyPrice * 100) -
self.commision
) / 100 #account for commision when you originally bought the stock
completedBuySellCyles.append([
ticker, buyDate,
float(buyPrice), date,
float(openPrice), quantity, self.commision / 100,
tradeProfit
])
else:
raise ValueError(
"Ticker {} is not a currently owned stock".format(ticker))
logAdditions = pd.DataFrame(executedOrders,
columns=self.dailyLogColumns)
self.dailyLogs = self.dailyLogs.append(logAdditions, ignore_index=True)
tradeHistoryAdditions = pd.DataFrame(completedBuySellCyles,
columns=self.tradeHistoryColumns)
self.tradeHistory = self.tradeHistory.append(tradeHistoryAdditions,
ignore_index=True)
def placeBuyOrders(self, listOfOrders, date, simulation=True):
'''
places Sell Order for every order in listOfOrders.
Each element in listOfOrders is a list in the following format:
[<str ticker>, <int quantity>]
Date is the current date, in string format "YYYY-MM-DD".
Setting simulation to False will actually execute the buy orders (not yet implimented)
'''
if (not simulation):
pass #actually execute buy orders
action = "Buy"
executedOrders = []
for order in listOfOrders:
ticker = order[0]
quantity = order[1]
if (quantity > 0):
tickerData = database.getDataframe(ticker)
openPrice = tickerData.loc[date, "Open"]
self.balance -= self.commision
tradeCost = int(openPrice * quantity * 100)
if ((self.balance - tradeCost) >= 0):
#There is enough money for the trade
self.balance -= tradeCost #*100 since balance is in cents
if (ticker in self.stocksOwned):
#If we own the stock, add the new quantity to the old quantity
self.stocksOwned[ticker] = self.stocksOwned[ticker][
0] + quantity, openPrice, date, openPrice #each element in stocksOwned is ticker: (quantity, buyPrice, dateOfPurchase, mostRecentPrice)
else:
self.stocksOwned[
ticker] = quantity, openPrice, date, openPrice #each element in stocksOwned is ticker: (quantity, buyPrice, dateOfPurchase, mostRecentPrice)
if (simulation):
self.updateAssets(date)
else:
self.updateAssets()
totalAssets = float(self.balance + self.stockAssets) / 100
timeStamp = time.time()
timeOfExecution = datetime.datetime.fromtimestamp(
timeStamp).strftime('%Y-%m-%d %H:%M:%S')
executedOrders.append([
date, ticker, quantity,
float(openPrice), action, timeOfExecution, totalAssets,
float(self.balance) / 100,
float(self.stockAssets) / 100
])
else:
raise ValueError(
"Insufficient funds to execute trade {}".format(order))
else:
raise ValueError("Buy quantity of 0 recieved")
self.updateAssets(date)
totalAssets = float(self.balance + self.stockAssets) / 100
executedOrders.append([
date, "", "", "", "CHECKPOINT",
datetime.datetime.fromtimestamp(
time.time()).strftime('%Y-%m-%d %H:%M:%S'), totalAssets,
float(self.balance) / 100,
float(self.stockAssets) / 100
])
logAdditions = pd.DataFrame(executedOrders,
columns=self.dailyLogColumns)
self.dailyLogs = self.dailyLogs.append(logAdditions, ignore_index=True)
self.updateAssets(date)
def selectStocksToBuy(self, maxNumberOfStocks, date=False, customTickerList=False, genricParameters=[]):
'''
Selects which stocks to buy, and the proportion of funds to be allocated to each.
maximumNumberOfStocks is the highest number of stocks the caller wants to select.
Passing a date parameter will select stocks for the given date. Ommitting it will
select stocks based on realtime data.
Passing a customTickerList will only analyze tickers included in that list.
Returns a list of in the following format: [ [Ticker1, RatioOfFundAllocation1], [Ticker2, RatioOfFundAllocation2], ... ] .
All ratios will add up to 1.
'''
if (not date):
raise ValueError("Real time selection not availible for this selector")
else:
if (customTickerList):
tickerList = customTickerList
else:
tickerList = database.getTickerList()
argList = []
for ticker in tickerList:
argList.append([ticker, date])
processCount = multiprocessing.cpu_count()-1
processPool = Pool(processCount)
results = list(processPool.map(self.getProfitSpeed, argList))
processPool.close()
processPool.join()
for i in range(len(results)-1, -1, -1):
if (results[i]):
pass
else:
del results[i]
results = sorted(results)
#print(results)
randomnessFactor = 0.85 #Used to randomize selection so that the selector isn't perfect. 0 is perfect, 1 is completely random
lengthOfResults = len(results)
#print(lengthOfResults)
indexesOfChosenStocks = []
for i in range(0, maxNumberOfStocks, 1):
if (lengthOfResults-len(indexesOfChosenStocks)>0):
#There are still stocks left to chose from
while (True):
indexToUse = randint(int(randomnessFactor*(lengthOfResults-1)), lengthOfResults-1)
if (indexToUse in indexesOfChosenStocks):
pass
else:
indexesOfChosenStocks.append(indexToUse)
break
else:
#There are no more stocks left to select
break
#print(indexesOfChosenStocks)
stocksToConsider = []
for index in indexesOfChosenStocks:
stocksToConsider.append(results[index])
randomRatios = list(np.random.dirichlet(np.ones(maxNumberOfStocks),size=1)[0]) #https://stackoverflow.com/questions/18659858/generating-a-list-of-random-numbers-summing-to-1
stocksToBuy = []
for i in range(0, len(stocksToConsider), 1):
stocksToBuy.append([stocksToConsider[i][1], randomRatios[i]])
return stocksToBuy
positiveProfitSpeedList = positiveProfitSpeedList[0:numberOfNegatives]
elif (numberOfNegatives > numberOfPositives):
negativeProfitSpeedList = negativeProfitSpeedList[0:numberOfPositives]
DataPoints += positiveProfitSpeedList
DataPoints += negativeProfitSpeedList
return DataPoints
except Exception as e:
#print(e)
return False
if __name__ == '__main__':
stockData = database.getDataframe("TSLA")
print(stockData)
print(stockData["2018-06-14"])
i = stockData.index.get_loc("2018-06-14")[0]
print(i)
data = stockData.iloc[i:i+90]["Profit Speed"]
print(data)
print(len(data))
'''
#Populate datapoints list
numberOfStocks = 2000
tickerList = database.getTickerList(randomize=True)[0:numberOfStocks]
print("Number of stocks for training: " + str(numberOfStocks))
print("Getting datapoints for training...")
processCount = multiprocessing.cpu_count() - 1