def parseQuandl(strResponse):
"""
Parse a quandl GET request to collect all the tickers returned in the response
:param strResponse: a response from quandl
:return: array containing tickers
"""
aTickers = []
aRows = strResponse.split('\n')
#the first row will be a header
#find the 'ticker' column so we can figure out what column contains the ticker
i = 0
iTickerCol = -1
aHeader = aRows.pop(0).split(',')
while i < len(aHeader):
if aHeader[i] == 'ticker':
iTickerCol = i
break
i += 1
if iTickerCol == -1:
Logger.logError('There were no tickers returned from quandl')
#loop through the remaining rows and collect all the tickers
for strRow in aRows:
aRow = strRow.split(',')
aTickers.append(aRow[iTickerCol])
return aTickers
def __init__(self):
self.aPortfolios = []
# recursively iterate over the directory that contains algorithm classes
for root, dirs, files in os.walk("Algorithms"):
for file in files:
if file.endswith(".py") and file != "__init__.py":
try:
# get an algorithm's file information
path = os.path.join(root, file)
info = os.stat(path)
oPortfolio = {"last_modified": info.st_mtime, "file_path": path, "file_name": file}
# TODO: we should also add a user identifier so we know who this algo belongs to
# get an algorithm's object instance
strAlgorithmClass = file.split(".")[0]
oModule = imp.load_source(strAlgorithmClass, path)
oAlgorithm = getattr(oModule, strAlgorithmClass)()
# store an algorithm's file info and obj instance
oPortfolio['algorithm'] = oAlgorithm
self.aPortfolios.append(oPortfolio)
del path, info, strAlgorithmClass, oModule
except Exception as e:
Logger.logError("Failed to instantiate {0}: {1}".format(str(file), str(e)))
def get(oDB, strTableTemplate, aTickers, strDate):
aRetval = {}
for strTicker in aTickers:
#dont do the same work twice
if strTicker in aRetval:
continue
strTable = strTableTemplate.replace(TABLE_WILDCARD, strTicker)
strQuery = """
SELECT *
FROM {0}
WHERE date >= {1}
ORDER BY date DESC;
""".format(strTable, quoteString(strDate))
#go to next ticker if error selecting data
aData = Connection.execute(oDB, strQuery)
if not aData:
Logger.logError("Error trying to select data")
continue
#add data to retval
aRetval[strTicker] = mapSelect(aData)
return aRetval
def get(strRequest):
"""
Performs a GET request provided by the given request
:param strRequest: the URL of the request
:return: response if OK, False if not
"""
try:
return urllib2.urlopen(strRequest);
except urllib2.HTTPError as e:
strError = "The server couldn't fulfill the request. Error code: " + str(e.code)
Logger.logError(strError)
return None
except urllib2.URLError as e:
strError = "We failed to reach the server. Reason: " + e.reason
Logger.logError(strError)
return None
def insertDailyData(self):
"""
Routine for collecting and inserting daily data from the YahooApi. All data is for the previously closed
trading day.
:return: None
"""
Logger.logApp("Collecting and inserting daily data...")
# chunk the tickers into a managable size, retrieve data for each chunk, and then insert each chunk
# chunking allows us to insert periodicly through the data collection process and ensures our YahooApi request
# doesnt return a 414 response code (URI too long)
iCurChunk = 0
aTickers = self.getQuandlTickers(AppVars.DATA_DAILY_TICKERS)
aTickerChunks = Utils.chunk(aTickers, AppVars.CHUNK_TICKERS)
for iCurChunk in range(0, len(aTickerChunks)):
oData = Api.getData(aTickerChunks[iCurChunk], AppVars.DATA_DAILY_DIMENSIONS)
if oData:
TradingData.insert(self.oDB, TradingData.S_DAILY_DATA, oData)
self.oDB.commit()
Logger.logApp("Inserting data for chunk " + str(iCurChunk + 1) + " of " + str(len(aTickerChunks)))
else:
Logger.logError('There was an error retrieving data for chunk ' + str(iCurChunk + 1))
del oData
def run(self):
"""
Main daemon process invoked by DataDaemon. This method is a infinite loop that has logic in it's body to
execute commands at specific times of day. More specifically, this process is responsible for creating,
running, and closing each trading day. This process will get killed when the daemon stops.
:return:
"""
# service variables
bTrading = False
while True:
# Get the current EST time and date
oNow = datetime.datetime.now(timezone(Conf.MARKET_TIMEZONE))
oNowDate = datetime.datetime(oNow.year, oNow.month, oNow.day)
# Market is only open on week days from 9:30AM EST to 4:00PM EST
bIsWeekDay = not(oNow.strftime('%A') == 'sunday' or oNow.strftime('%A') == 'saturday')
bIsMarketHours = datetime.time(Conf.MARKET_OPEN_HOUR, Conf.MARKET_OPEN_MINUTE) <= datetime.time(oNow.hour, oNow.minute) \
and datetime.time(oNow.hour, oNow.minute) < datetime.time(Conf.MARKET_CLOSE_HOUR, Conf.MARKET_CLOSE_MINUTE)
bIsOpen = bIsWeekDay and bIsMarketHours
# it's after 5:00AM EST on a week day, let's collect the previous days data and get everything set up
if (bIsWeekDay and not bTrading and oNow.hour >= 5) or Conf.DAEMON_IS_DEBUG:
# insert daily data from yesterday
if Conf.DAEMON_INSERT_DAILY:
self.insertDailyData()
# market vars, must be deleted at EOD
aTickers = self.getQuandlTickers(AppVars.DATA_RT_TICKERS)
aTickerChunks = Utils.chunk(aTickers, AppVars.CHUNK_TICKERS)
del aTickers
oPortfolioCollection = PortfolioCollection()
# OK to stop trading
bTrading = True
# the market is open! start collecting data and trading
if (bTrading and bIsOpen and aTickerChunks) or Conf.DAEMON_IS_DEBUG:
Logger.logApp("Starting a trading cycle...")
# get current pricing data for all tickers and create a data map where keys are tickers and values are
# the location of the ticker's value in the data list
aDataList = []
oDataMap = {}
for iCurChunk in range(0, len(aTickerChunks)):
aChunkData = Api.getData(aTickerChunks[iCurChunk], AppVars.DATA_RT_DIMENSIONS)
for iDataIndex in range(len(aDataList), len(aDataList) + len(aChunkData)):
oDataMap[aChunkData[iDataIndex - len(aDataList)][Company.SYMBOL]] = iDataIndex
aDataList += aChunkData
del aChunkData
del iCurChunk
del iDataIndex
# broadcast new data to all portfolios
for oPortfolio in oPortfolioCollection.iteritems():
oAlgorithm = oPortfolio['algorithm']
oAlgorithm.run(oDataMap)
# insert new data
if aDataList:
TradingData.insert(self.oDB, TradingData.S_RT_DATA, aDataList)
self.oDB.commit()
else:
Logger.logError('There was an error inserting real time data')
del oDataMap
Logger.logApp("Finished a trading cycle")
# it's after 4:30PM EST on a week day let's close the trading day and go to sleep
if (bIsWeekDay and bTrading and oNow.hour >= 16 and oNow.minute > 30) or Conf.DAEMON_IS_DEBUG:
# insert portfolio data
for oPortfolio in oPortfolioCollection.iteritems():
oAlgorithm = oPortfolio['algorithm']
oAlgorithm.insert()
# clean up market vars
del aTickerChunks
del oPortfolioCollection
# OK to start trading
bTrading = False
time.sleep(Conf.DAEMON_SLEEP)
def insert(oDB, strTableTemplate, aRows):
"""
Inserts data into the tables of type strTableTemplate. Will create tables if needed. Each row must contain a
Stock.DATE and Company.SYMBOL key. Each row represents a unique
INSERT IGNORE INTO strTableTemplate
(date, dim_name, value)
VALUES (aRows[Stock.DATE], aRows[Dimension], aRows[DimensionValue])
:param oDB: MySQLdb object
:param strTableTemplate: Type of table that will recieve inserts
:param aRows: An array of objects where the object keys are Dimensions
:return: boolean indicating the success of the inserts
"""
bSuccess = True
strColumns = '(' + ",".join(['date', 'dim_name', 'value']) + ')'
while aRows:
oRow = aRows.pop()
strDateDim = ''
strDateDimVal = ''
if Stock.DATE in oRow and oRow[Stock.DATE] != 'N/A':
strDateDim = Stock.DATE
strDateDimVal = oRow[Stock.DATE]
elif RealTime.RT_LAST_TRADE in oRow and oRow[RealTime.RT_LAST_TRADE] != 'N/A':
strDateDim = RealTime.RT_LAST_TRADE
strDateDimVal = oRow[RealTime.RT_LAST_TRADE]
else:
continue
if not Company.SYMBOL in oRow and oRow[Company.SYMBOL] != 'N/A':
continue
strDate = datetime.datetime.strptime(strDateDimVal.replace('"', ''), '%m/%d/%Y').strftime('%Y-%m-%d')
strSymbol = oRow[Company.SYMBOL]
strTable = strTableTemplate.replace(TABLE_WILDCARD, strSymbol).replace('"', '')
#create a table for this stock if it doesnt exist. Skip insert if there's a MySQL error
if not createTable(oDB, strTable):
bSuccess = False
continue
#insert
for oDim, mVal in oRow.iteritems():
#never insert the date dimension or any dimension with a 'N/A' value
if oDim == strDateDim or mVal == 'N/A':
continue
#construct and execute INSERT statement
strRow = '(' + ",".join([quoteString(strDate), quoteString(oDim), quoteString(mVal)]) + ')'
strInsert = """
INSERT IGNORE INTO {0}
{1}
VALUES
{2};
""".format(strTable, strColumns, strRow)
if not Connection.insert(oDB, strInsert):
Logger.logError("Failed to execute: " + strInsert)
bSuccess = False
return bSuccess