def main():
util.check_include()
util.set_log_file()
#get last calcres of previous day
prevDay = util.exchangeTradingOffset(os.environ['PRIMARY_EXCHANGE'],
os.environ['DATE'], -1)
fs = file_source.FileSource(os.environ['RUN_DIR'] + '/../' + str(prevDay) +
'/calcres')
calcresFiles = fs.list(r'calcres.*\.txt\.gz')
if len(calcresFiles) == 0:
util.error("Failed to locate calcres file")
sys.exit(1)
calcresFiles.sort(key=lambda x: x[0], reverse=True)
lastCalcresFile = os.environ['RUN_DIR'] + '/../' + str(
prevDay) + '/calcres/' + calcresFiles[0][0]
secidParams = {}
for line in GzipFile(lastCalcresFile, 'r'):
if line.startswith('FCOV'): continue
secid, name, datatype, datetime, value, currency, born = line.split(
'|')
if int(secid) not in secidParams:
secidParams[int(secid)] = {}
if name == 'F:BBETA':
secidParams[int(secid)]['BBETA'] = float(value)
elif name == 'F:ASE_BETA90':
secidParams[int(secid)]['ASE_BETA'] = float(value)
elif name == 'CAPITALIZATION':
secidParams[int(secid)]['CAP'] = float(value)
#get tickers
tic2sec, sec2tic = datafiles.load_tickers(os.environ['RUN_DIR'] +
'/tickers.txt')
etfbetaFilename = os.environ['RUN_DIR'] + '/exec/etfbetafile'
etfbetaFile = open(etfbetaFilename, 'w')
etfbetaFile.write('#ETF BETA MKT-CAP\n')
tickers = tic2sec.keys()
tickers.sort()
count = 0
for ticker in tickers:
secid = tic2sec.get(ticker, None)
if secid not in secidParams:
continue
bbeta = secidParams[secid].get('BBETA', None)
asebeta = secidParams[secid].get('ASE_BETA', None)
mcap = secidParams[secid].get('CAP', None)
if bbeta is None or asebeta is None or mcap is None:
util.error(
'Error while getting data for secid {}: ticker={}, bbeta={}, asebeta={}, mcap={}'
.format(secid, ticker, bbeta, asebeta, mcap))
continue
beta = 0.5 * (bbeta + asebeta)
etfbetaFile.write('{0},SPY,{1:.3f},{2:.3f}\n'.format(
ticker, beta, mcap))
count += 1
etfbetaFile.close()
print 'Finished writing etfbeta file: {} for {} tickers'.format(
etfbetaFilename, count)
#!/usr/bin/env python
import os
import sys
import util
from gzip import GzipFile
from data_sources import file_source
import datafiles
ADV_FRACTION = 1.3 / 100.0
LOT_SIZE = 100
MIN_REQUEST = 5000
MAX_DOLLARS = 1.5e7
if __name__ == "__main__":
util.check_include()
util.set_log_file()
#get last calcres of day
fs = file_source.FileSource(os.environ["RUN_DIR"] + "/calcres")
calcresFiles = fs.list(r'calcres.*\.txt\.gz')
if len(calcresFiles) == 0:
util.error("Failed to locate calcres file")
sys.exit(1)
calcresFiles.sort(key=lambda x: x[0], reverse=True)
lastCalcresFile = os.environ["RUN_DIR"] + "/calcres/" + calcresFiles[0][0]
#get tradeable secs and advp
tradeable = set()
advps = {}
prices = {}
def main():
util.check_include()
util.set_log_file()
tickersFile = open(os.environ['RUN_DIR'] + '/tickers.txt', 'r')
tickerLines = [line.strip().split('|') for line in tickersFile]
tickersFile.close()
sec2tic = {}
tic2sec = {}
for line in tickerLines:
(ticker, secid) = (line[0], int(line[1]))
sec2tic[secid] = ticker
tic2sec[ticker] = secid
prevDay = util.exchangeTradingOffset(os.environ['PRIMARY_EXCHANGE'],
os.environ['DATE'], -1)
tradeSzFile = open(
os.environ['DATA_DIR'] + '/bars/' + str(prevDay) + '/tradeSz.txt', 'r')
tradeSzLines = [line.strip().split('|') for line in tradeSzFile]
tradeSzFile.close()
dataBySecID = {}
allSizes = {}
for line in tradeSzLines[1:]:
(secid, size, count) = (int(line[0]), int(line[1]), int(line[2]))
if secid not in dataBySecID:
dataBySecID[secid] = []
dataBySecID[secid].append((size, count))
if size not in allSizes:
allSizes[size] = 0
allSizes[size] += count
klist = {}
for secid in sec2tic.keys():
if secid not in dataBySecID:
klist[secid] = (-2, -2, -2, -2)
continue
dataBySecID[secid].sort(key=lambda x: x[1])
keys = []
values = []
scaledKeys = []
logKeys = []
logValues = []
for (key, value) in dataBySecID[secid]:
if (key < 100) or (key > 10000): continue
keys.append(key)
values.append(value)
scaledKeys.append(key / 100.0)
logKeys.append(log(key))
logValues.append(log(value))
klist[secid] = (-1, -1, -1, -1)
for ii in range(-3, -1 * (len(keys) + 1), -1):
(a, b, R2) = linreg(logKeys[ii:], logValues[ii:])
if R2 == 1:
print 'R2 = 1 for ' + sec2tic[secid] + ' for ' + str(
-1 * ii) + ' out of ' + str(len(keys))
if R2 > 0.9:
klist[secid] = (a, R2, -1 * ii, len(keys))
else:
break
# read previous day's klist file and compute an average
prevKlistFile = open(
os.environ['RUN_DIR'] + '/../' + str(prevDay) + '/exec/klist', 'r')
prevKlistLines = [line.strip().split(',') for line in prevKlistFile]
prevKlistFile.close()
prevKlist = {}
# ignore header
for line in prevKlistLines[1:]:
prevKlist[line[0]] = float(line[1])
klistFilename = os.environ['RUN_DIR'] + '/exec/klist'
outFile = open(klistFilename, 'w')
#outFile.write('Stock,K,R2,Reg,Tot\n')
outFile.write('Stock,K,R2\n')
tickers = tic2sec.keys()
tickers.sort()
count = 0
notInPrev = 0
for ticker in tickers:
secid = tic2sec.get(ticker, None)
if secid not in klist:
continue
kPoints = klist[secid][2]
if kPoints < 0:
continue
kValue = klist[secid][0]
if ticker in prevKlist:
kValue = 0.5 * (kValue + prevKlist[ticker])
else:
notInPrev += 1
if kValue > -1:
util.error(
'For {}, kValue = {}, learned today = {}, R2 = {}, points for today = {} out of {}\n'
.format(ticker, kValue, klist[secid][0], klist[secid][1],
klist[secid][2], klist[secid][3]))
#outFile.write('{0},{1:.4f},{2:.4f}{3}{4}\n'.format(ticker, klist[secid][0], kValue, klist[secid][2], klist[secid][3])
outFile.write('{0},{1:.4f},{2:.4f}\n'.format(ticker, kValue,
klist[secid][1]))
count += 1
outFile.close()
print 'Finished writing klist file: {} for {} tickers. {} tickers not present in previous klist file.'.format(
klistFilename, count, notInPrev)
