def findEvents(symbols, startday,endday,verbose=False):
timeofday=dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday,endday,timeofday)
dataobj = da.DataAccess('Norgate')
if verbose:
print __name__ + " reading data"
adjclose = dataobj.get_data(timestamps, symbols, closefield)
adjclose = (adjclose.fillna()).fillna(method='backfill')
if verbose:
print __name__ + " finding events"
# for symbol in symbols:
# close[symbol][close[symbol]>= 1.0] = np.NAN
# for i in range(1,len(close[symbol])):
# if np.isnan(close[symbol][i-1]) and close[symbol][i] < 1.0 :#(i-1)th was > $1, and (i)th is <$1
# close[symbol][i] = 1.0 #overwriting the price by the bit
# close[symbol][close[symbol]< 1.0] = np.NAN
#print adjclose
# Get the 20 day moving avg and moving stddev
movavg = pa.rolling_mean(adjclose,20,min_periods=20)
movstddev = pa.rolling_std(adjclose, 20, min_periods=20)
# Compute the upper and lower bollinger bands
upperband = movavg + 2*movstddev
lowerband = movavg - 2*movstddev
#bandwidth = (upperband - lowerband)/movavg
#print bandwidth
#print upperband
# Compute the event matrix as follows:
# Set periods of low volatility to 1
# In from the period of low volatility to the period of say, 15 days, following low volatility
# if the stock price breaks above the upper band, there is a surge. this is a positive event. Set this event to 2
# Finally, set all events other than 2 to NaN. Then, set all 2's to 1
lookaheadperiod = 15
eventMatrix = adjclose.copy()
for symbol in symbols:
for row in range(len(adjclose[:][symbol])):
eventMatrix[symbol][row] = np.NAN
if upperband[symbol][row] > 0 and lowerband[symbol][row] > 0 and movavg[symbol][row] > 0:
if (upperband[symbol][row] - lowerband[symbol][row])/movavg[symbol][row] < 0.10:
eventMatrix[symbol][row] = 1
else:
currow = row - 1
numOnes = 0
while currow > row - lookaheadperiod and currow >= 0:
if eventMatrix[symbol][currow] != 1:
break
if eventMatrix[symbol][currow] == 1 and adjclose[symbol][row] > upperband[symbol][row]:
numOnes = numOnes + 1
currow = currow - 1
if numOnes >= 5:
eventMatrix[symbol][row] = 2
eventMatrix[symbol][eventMatrix[symbol]!= 2] = np.NAN
eventMatrix[symbol][eventMatrix[symbol]== 2] = 1
return eventMatrix
def log500( sLog ):
'''
@summary: Loads cached features.
@param sLog: Filename of features.
@return: Nothing, logs features to desired location
'''
lsSym = ['A', 'AA', 'AAPL', 'ABC', 'ABT', 'ACE', 'ACN', 'ADBE', 'ADI', 'ADM', 'ADP', 'ADSK', 'AEE', 'AEP', 'AES', 'AET', 'AFL', 'AGN', 'AIG', 'AIV', 'AIZ', 'AKAM', 'AKS', 'ALL', 'ALTR', 'AMAT', 'AMD', 'AMGN', 'AMP', 'AMT', 'AMZN', 'AN', 'ANF', 'ANR', 'AON', 'APA', 'APC', 'APD', 'APH', 'APOL', 'ARG', 'ATI', 'AVB', 'AVP', 'AVY', 'AXP', 'AZO', 'BA', 'BAC', 'BAX', 'BBBY', 'BBT', 'BBY', 'BCR', 'BDX', 'BEN', 'BF.B', 'BHI', 'BIG', 'BIIB', 'BK', 'BLK', 'BLL', 'BMC', 'BMS', 'BMY', 'BRCM', 'BRK.B', 'BSX', 'BTU', 'BXP', 'C', 'CA', 'CAG', 'CAH', 'CAM', 'CAT', 'CB', 'CBG', 'CBS', 'CCE', 'CCL', 'CEG', 'CELG', 'CERN', 'CF', 'CFN', 'CHK', 'CHRW', 'CI', 'CINF', 'CL', 'CLF', 'CLX', 'CMA', 'CMCSA', 'CME', 'CMG', 'CMI', 'CMS', 'CNP', 'CNX', 'COF', 'COG', 'COH', 'COL', 'COP', 'COST', 'COV', 'CPB', 'CPWR', 'CRM', 'CSC', 'CSCO', 'CSX', 'CTAS', 'CTL', 'CTSH', 'CTXS', 'CVC', 'CVH', 'CVS', 'CVX', 'D', 'DD', 'DE', 'DELL', 'DF', 'DFS', 'DGX', 'DHI', 'DHR', 'DIS', 'DISCA', 'DNB', 'DNR', 'DO', 'DOV', 'DOW', 'DPS', 'DRI', 'DTE', 'DTV', 'DUK', 'DV', 'DVA', 'DVN', 'EBAY', 'ECL', 'ED', 'EFX', 'EIX', 'EL', 'EMC', 'EMN', 'EMR', 'EOG', 'EP', 'EQR', 'EQT', 'ERTS', 'ESRX', 'ETFC', 'ETN', 'ETR', 'EW', 'EXC', 'EXPD', 'EXPE', 'F', 'FAST', 'FCX', 'FDO', 'FDX', 'FE', 'FFIV', 'FHN', 'FII', 'FIS', 'FISV', 'FITB', 'FLIR', 'FLR', 'FLS', 'FMC', 'FO', 'FRX', 'FSLR', 'FTI', 'FTR', 'GAS', 'GCI', 'GD', 'GE', 'GILD', 'GIS', 'GLW', 'GME', 'GNW', 'GOOG', 'GPC', 'GPS', 'GR', 'GS', 'GT', 'GWW', 'HAL', 'HAR', 'HAS', 'HBAN', 'HCBK', 'HCN', 'HCP', 'HD', 'HES', 'HIG', 'HNZ', 'HOG', 'HON', 'HOT', 'HP', 'HPQ', 'HRB', 'HRL', 'HRS', 'HSP', 'HST', 'HSY', 'HUM', 'IBM', 'ICE', 'IFF', 'IGT', 'INTC', 'INTU', 'IP', 'IPG', 'IR', 'IRM', 'ISRG', 'ITT', 'ITW', 'IVZ', 'JBL', 'JCI', 'JCP', 'JDSU', 'JEC', 'JNJ', 'JNPR', 'JNS', 'JOYG', 'JPM', 'JWN', 'K', 'KEY', 'KFT', 'KIM', 'KLAC', 'KMB', 'KMX', 'KO', 'KR', 'KSS', 'L', 'LEG', 'LEN', 'LH', 'LIFE', 'LLL', 'LLTC', 'LLY', 'LM', 'LMT', 'LNC', 'LO', 'LOW', 'LSI', 'LTD', 'LUK', 'LUV', 'LXK', 'M', 'MA', 'MAR', 'MAS', 'MAT', 'MCD', 'MCHP', 'MCK', 'MCO', 'MDT', 'MET', 'MHP', 'MHS', 'MJN', 'MKC', 'MMC', 'MMI', 'MMM', 'MO', 'MOLX', 'MON', 'MOS', 'MPC', 'MRK', 'MRO', 'MS', 'MSFT', 'MSI', 'MTB', 'MU', 'MUR', 'MWV', 'MWW', 'MYL', 'NBL', 'NBR', 'NDAQ', 'NE', 'NEE', 'NEM', 'NFLX', 'NFX', 'NI', 'NKE', 'NOC', 'NOV', 'NRG', 'NSC', 'NTAP', 'NTRS', 'NU', 'NUE', 'NVDA', 'NVLS', 'NWL', 'NWSA', 'NYX', 'OI', 'OKE', 'OMC', 'ORCL', 'ORLY', 'OXY', 'PAYX', 'PBCT', 'PBI', 'PCAR', 'PCG', 'PCL', 'PCLN', 'PCP', 'PCS', 'PDCO', 'PEG', 'PEP', 'PFE', 'PFG', 'PG', 'PGN', 'PGR', 'PH', 'PHM', 'PKI', 'PLD', 'PLL', 'PM', 'PNC', 'PNW', 'POM', 'PPG', 'PPL', 'PRU', 'PSA', 'PWR', 'PX', 'PXD', 'QCOM', 'QEP', 'R', 'RAI', 'RDC', 'RF', 'RHI', 'RHT', 'RL', 'ROK', 'ROP', 'ROST', 'RRC', 'RRD', 'RSG', 'RTN', 'S', 'SAI', 'SBUX', 'SCG', 'SCHW', 'SE', 'SEE', 'SHLD', 'SHW', 'SIAL', 'SJM', 'SLB', 'SLE', 'SLM', 'SNA', 'SNDK', 'SNI', 'SO', 'SPG', 'SPLS', 'SRCL', 'SRE', 'STI', 'STJ', 'STT', 'STZ', 'SUN', 'SVU', 'SWK', 'SWN', 'SWY', 'SYK', 'SYMC', 'SYY', 'T', 'TAP', 'TDC', 'TE', 'TEG', 'TEL', 'TER', 'TGT', 'THC', 'TIE', 'TIF', 'TJX', 'TLAB', 'TMK', 'TMO', 'TROW', 'TRV', 'TSN', 'TSO', 'TSS', 'TWC', 'TWX', 'TXN', 'TXT', 'TYC', 'UNH', 'UNM', 'UNP', 'UPS', 'URBN', 'USB', 'UTX', 'V', 'VAR', 'VFC', 'VIA.B', 'VLO', 'VMC', 'VNO', 'VRSN', 'VTR', 'VZ', 'WAG', 'WAT', 'WDC', 'WEC', 'WFC', 'WFM', 'WFR', 'WHR', 'WIN', 'WLP', 'WM', 'WMB', 'WMT', 'WPI', 'WPO', 'WU', 'WY', 'WYN', 'WYNN', 'X', 'XEL', 'XL', 'XLNX', 'XOM', 'XRAY', 'XRX', 'YHOO', 'YUM', 'ZION', 'ZMH']
lsSym.append('SPY')
lsSym.sort()
''' Max lookback is 6 months '''
dtEnd = dt.datetime.now()
dtEnd = dtEnd.replace(hour=16, minute=0, second=0, microsecond=0)
dtStart = dtEnd - relativedelta(months=6)
''' Pull in current data '''
norObj = da.DataAccess('Norgate')
''' Get 2 extra months for moving averages and future returns '''
ldtTimestamps = du.getNYSEdays( dtStart - relativedelta(months=2), \
dtEnd + relativedelta(months=2), dt.timedelta(hours=16) )
dfPrice = norObj.get_data( ldtTimestamps, lsSym, 'close' )
dfVolume = norObj.get_data( ldtTimestamps, lsSym, 'volume' )
''' Imported functions from qstkfeat.features, NOTE: last function is classification '''
lfcFeatures, ldArgs, lsNames = getFeatureFuncs()
''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
applyFeatures( dfPrice, dfVolume, lfcFeatures, ldArgs, sLog=sLog )
def findEvents(symbols, startday,endday,verbose=False):
timeofday=dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday,endday,timeofday)
dataobj = da.DataAccess('Norgate')
if verbose:
print __name__ + " reading data"
adjclose = dataobj.get_data(timestamps, symbols, closefield)
adjclose = (adjclose.fillna()).fillna(method='backfill')
adjcloseSPY = dataobj.get_data(timestamps, ['SPY'], closefield)
adjcloseSPY = (adjcloseSPY.fillna()).fillna(method='backfill')
if verbose:
print __name__ + " finding events"
# for symbol in symbols:
# close[symbol][close[symbol]>= 1.0] = np.NAN
# for i in range(1,len(close[symbol])):
# if np.isnan(close[symbol][i-1]) and close[symbol][i] < 1.0 :#(i-1)th was > $1, and (i)th is <$1
# close[symbol][i] = 1.0 #overwriting the price by the bit
# close[symbol][close[symbol]< 1.0] = np.NAN
#print adjclose
# Get the 20 day moving avg and moving stddev
movavg = pa.rolling_mean(adjclose,20,min_periods=20)
movavgSPY = pa.rolling_mean(adjcloseSPY,20,min_periods=20)
movstddev = pa.rolling_std(adjclose, 20, min_periods=20)
movstddevSPY = pa.rolling_std(adjcloseSPY, 20, min_periods=20)
upperband = movavg + 2*movstddev
upperbandSPY = movavgSPY + 2*movstddevSPY
lowerband = movavg - 2*movstddev
lowerbandSPY = movavgSPY - 2*movstddevSPY
# Compute the bollinger %b indicator for all stocks
normalizedindicator = 2*(adjclose - movavg)/(upperband - lowerband)
#print normalizedindicator
normalizedindicatorSPY = 2*(adjcloseSPY - movavgSPY)/(upperbandSPY - lowerbandSPY)
#print normalizedindicatorSPY
#bandwidth = (upperband - lowerband)/movavg
#print bandwidth
#print upperband
# Compute the event matrix as follows:
# Set periods of low volatility to 1
# In from the period of low volatility to the period of say, 15 days, following low volatility
# if the stock price breaks above the upper band, there is a surge. this is a positive event. Set this event to 2
# Finally, set all events other than 2 to NaN. Then, set all 2's to 1
eventMatrix = adjclose.copy()
for symbol in symbols:
for row in range(len(adjclose[:][symbol])):
eventMatrix[symbol][row] = np.NAN
if normalizedindicator[symbol][row] - normalizedindicatorSPY['SPY'][row] >= 0.5:
eventMatrix[symbol][row] = 1
return eventMatrix
def findEvents(symbols, startday,endday,verbose=False):
timeofday=dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday,endday,timeofday)
dataobj = da.DataAccess('Norgate')
if verbose:
print __name__ + " reading data"
close = dataobj.get_data(timestamps, symbols, closefield)
close = (close.fillna()).fillna(method='backfill')
if verbose:
print __name__ + " finding events"
for symbol in symbols:
close[symbol][close[symbol]>= 1.0] = np.NAN
for i in range(1,len(close[symbol])):
if np.isnan(close[symbol][i-1]) and close[symbol][i] < 1.0 :#(i-1)th was > $1, and (i)th is <$1
close[symbol][i] = 1.0 #overwriting the price by the bit
close[symbol][close[symbol]< 1.0] = np.NAN
return close
def findEvents(symbols, startday, endday, verbose=False):
timeofday = dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday, endday, timeofday)
dataobj = da.DataAccess('Norgate')
if verbose:
print __name__ + " reading data"
close = dataobj.get_data(timestamps, symbols, closefield)
close = (close.fillna()).fillna(method='backfill')
if verbose:
print __name__ + " finding events"
for symbol in symbols:
close[symbol][close[symbol] >= 1.0] = np.NAN
for i in range(1, len(close[symbol])):
if np.isnan(
close[symbol][i - 1]
) and close[symbol][i] < 1.0: #(i-1)th was > $1, and (i)th is <$1
close[symbol][i] = 1.0 #overwriting the price by the bit
close[symbol][close[symbol] < 1.0] = np.NAN
return close
def __init__(self,eventMatrix,startday,endday,\
lookback_days = 20, lookforward_days =20,\
verbose=False):
""" Event Profiler class construtor
Parameters : evenMatrix
: startday
: endday
(optional) : lookback_days ( default = 20)
(optional) : lookforward_days( default = 20)
eventMatrix is a pandas DataMatrix
eventMatrix must have the following structure:
|IBM |GOOG|XOM |MSFT| GS | JP |
(d1)|nan |nan | 1 |nan |nan | 1 |
(d2)|nan | 1 |nan |nan |nan |nan |
(d3)| 1 |nan | 1 |nan | 1 |nan |
(d4)|nan | 1 |nan | 1 |nan |nan |
...................................
...................................
Also, d1 = start date
nan = no information about any event.
= status bit(positively confirms the event occurence)
"""
self.eventMatrix = eventMatrix
self.startday = startday
self.endday = endday
self.symbols = eventMatrix.columns
self.lookback_days = lookback_days
self.lookforward_days = lookforward_days
self.total_days = lookback_days + lookforward_days + 1
self.dataobj = da.DataAccess('Norgate')
self.timeofday = dt.timedelta(hours=16)
self.timestamps = du.getNYSEdays(startday,endday,self.timeofday)
self.verbose = verbose
if verbose:
print __name__ + " reading historical data"
self.close = self.dataobj.get_data(self.timestamps,\
self.symbols, "close", verbose=self.verbose)
self.close = (self.close.fillna()).fillna(method='backfill')
if __name__ == '__main__':
''' Use Dow 30 '''
lsSym = ['AA', 'AXP', 'BA', 'BAC', 'CAT', 'CSCO', 'CVX', 'DD', 'DIS', 'GE', 'HD', 'HPQ', 'IBM', 'INTC', 'JNJ', \
'JPM', 'KFT', 'KO', 'MCD', 'MMM', 'MRK', 'MSFT', 'PFE', 'PG', 'T', 'TRV', 'UTX', 'WMT', 'XOM' ]
#lsSym = ['XOM']
''' Get data for 2009-2010 '''
dtStart = dt.datetime(2010,8,01)
dtEnd = dt.datetime(2010,12,31)
norObj = da.DataAccess('Norgate')
ldtTimestamps = du.getNYSEdays( dtStart, dtEnd, dt.timedelta(hours=16) )
dfPrice = norObj.get_data( ldtTimestamps, lsSym, 'close' )
dfVolume = norObj.get_data( ldtTimestamps, lsSym, 'volume' )
''' Imported functions from qstkfeat.features, NOTE: last function is classification '''
lfcFeatures = [ featMA, featRSI, classFutRet ]
''' Default Arguments '''
#ldArgs = [{}] * len(lfcFeatures)
''' Custom Arguments '''
ldArgs = [ {'lLookback':30, 'bRel':True},\
{},\
{}]
def main():
# symbols = np.loadtxt('./Examples/Features/symbols.txt',dtype='S10',comments='#')
symbols = [
"AA",
"AXP",
"BA",
"BAC",
"CAT",
"CSCO",
"CVX",
"DD",
"DIS",
"GE",
"HD",
"HPQ",
"IBM",
"INTC",
"JNJ",
"JPM",
"KFT",
"KO",
"MCD",
"MMM",
"MRK",
"MSFT",
"PFE",
"PG",
"T",
"TRV",
"UTX",
"VZ",
"WMT",
"XOM",
]
# symbols = ['XOM']
# This is the start and end dates for the entire train and test data combined
alldatastartday = dt.datetime(2007, 1, 1)
alldataendday = dt.datetime(2010, 6, 30)
timeofday = dt.timedelta(hours=16)
timestamps = du.getNYSEdays(alldatastartday, alldataendday, timeofday)
dataobj = da.DataAccess("Norgate")
voldata = dataobj.get_data(timestamps, symbols, "volume", verbose=True)
voldata = (voldata.fillna()).fillna(method="backfill")
close = dataobj.get_data(timestamps, symbols, "close", verbose=True)
close = (close.fillna()).fillna(method="backfill")
featureList = [
featMA,
featMA,
featRSI,
featRSI,
featDrawDown,
featRunUp,
featVolumeDelta,
featVolumeDelta,
featAroon,
classFutRet,
]
featureListArgs = [
{"lLookback": 10, "bRel": True},
{"lLookback": 20},
{"lLookback": 10},
{"lLookback": 20},
{},
{},
{"lLookback": 10},
{"lLookback": 20},
{"bDown": False},
{"lLookforward": 5},
]
# print 'Applying Features'
#
# John Cornwell's featuretest.py was consulted for figuring out the syntax of ftu.applyFeatures() methods and ftu.stackSyms() methods
#
allfeatureValues = ftu.applyFeatures(close, voldata, featureList, featureListArgs)
trainstartday = dt.datetime(2007, 1, 1)
trainendday = dt.datetime(2009, 12, 31)
traintimestamps = du.getNYSEdays(trainstartday, trainendday, timeofday)
# print 'Stack Syms for Training'
trainingData = ftu.stackSyms(allfeatureValues, traintimestamps[0], traintimestamps[-1])
# print 'Norm Features for Training'
scaleshiftvalues = ftu.normFeatures(trainingData, -1.0, 1.0, False)
teststartday = dt.datetime(2010, 1, 1)
testendday = dt.datetime(2010, 6, 30)
testtimestamps = du.getNYSEdays(teststartday, testendday, timeofday)
# print 'Stack Syms for Test'
testData = ftu.stackSyms(allfeatureValues, testtimestamps[0], testtimestamps[-1])
# print 'Norm Features for Test'
ftu.normQuery(testData[:, :-1], scaleshiftvalues)
NUMFEATURES = 9
bestFeatureIndices = []
bestCorrelation = 0.0
fid = open("output.txt", "w")
for iteration in range(NUMFEATURES):
nextFeatureIndexToAdd = -1
for featureIndex in range(NUMFEATURES):
if featureIndex not in bestFeatureIndices:
bestFeatureIndices.append(featureIndex)
fid.write("testing feature set " + str(bestFeatureIndices) + "\n")
print("testing feature set " + str(bestFeatureIndices))
bestFeatureIndices.append(9)
curTrainingData = trainingData[:, bestFeatureIndices]
curTestData = testData[:, bestFeatureIndices]
bestFeatureIndices.remove(9)
kdtlearner = knn.kdtknn(5, "mean", leafsize=100)
kdtlearner.addEvidence(curTrainingData[:, :-1], curTrainingData[:, -1])
testEstimatedValues = kdtlearner.query(curTestData[:, :-1])
testcorrelation = np.corrcoef(testEstimatedValues.T, curTestData[:, -1].T)
curCorrelation = testcorrelation[0, 1]
fid.write("corr coef = %.4f\n" % (curCorrelation))
print("corr coef = %.4f" % (curCorrelation))
if curCorrelation > bestCorrelation:
nextFeatureIndexToAdd = featureIndex
bestCorrelation = curCorrelation
bestFeatureIndices.remove(featureIndex)
if nextFeatureIndexToAdd >= 0:
bestFeatureIndices.append(nextFeatureIndexToAdd)
else:
break
fid.write("best feature set is " + str(bestFeatureIndices) + "\n")
print("best feature set is " + str(bestFeatureIndices))
fid.write("corr coef = %.4f" % (bestCorrelation) + "\n")
print("corr coef = %.4f" % (bestCorrelation))
fid.close()
def getRandPort(
lNum,
dtStart=None,
dtEnd=None,
lsStocks=None,
dmPrice=None,
dmVolume=None,
bFilter=True,
fNonNan=0.95,
fPriceVolume=100 * 1000,
lSeed=None,
):
"""
@summary Returns a random portfolio based on certain criteria.
@param lNum: Number of stocks to be included
@param dtStart: Start date for portfolio
@param dtEnd: End date for portfolio
@param lsStocks: Optional list of ticker symbols, if not provided all symbols will be used
@param bFilter: If False, stocks are not filtered by price or volume data, simply return random Portfolio.
@param dmPrice: Optional price data, if not provided, data access will be queried
@param dmVolume: Optional volume data, if not provided, data access will be queried
@param fNonNan: Optional non-nan percent for filter, default is .95
@param fPriceVolume: Optional price*volume for filter, default is 100,000
@warning: Does not work for all sets of optional inputs, e.g. if you don't include dtStart, dtEnd, you need
to include dmPrice/dmVolume
@return list of stocks which meet the criteria
"""
if lsStocks is None:
if dmPrice is None and dmVolume is None:
norObj = da.DataAccess("Norgate")
lsStocks = norObj.get_all_symbols()
elif not dmPrice is None:
lsStocks = list(dmPrice.columns)
else:
lsStocks = list(dmVolume.columns)
if dmPrice is None and dmVolume is None and bFilter == True:
norObj = da.DataAccess("Norgate")
ldtTimestamps = du.getNYSEdays(dtStart, dtEnd, dt.timedelta(hours=16))
""" if dmPrice and dmVol are provided then we don't query it every time """
bPullPrice = False
bPullVol = False
if dmPrice is None:
bPullPrice = True
if dmVolume is None:
bPullVol = True
""" Default seed (none) uses system clock """
rand.seed(lSeed)
lsRetStocks = []
""" Loop until we have enough randomly selected stocks """
llRemainingIndexes = range(0, len(lsStocks))
lsValid = None
while len(lsRetStocks) != lNum:
lsCheckStocks = []
for i in range(lNum - len(lsRetStocks)):
lRemaining = len(llRemainingIndexes)
if lRemaining == 0:
print "Error in getRandPort: ran out of stocks"
return lsRetStocks
""" Pick a stock and remove it from the list of remaining stocks """
lPicked = rand.randint(0, lRemaining - 1)
lsCheckStocks.append(lsStocks[llRemainingIndexes.pop(lPicked)])
""" If bFilter is false, simply return our first list of stocks, don't check prive/vol """
if not bFilter:
return sorted(lsCheckStocks)
""" Get data if needed """
if bPullPrice:
dmPrice = norObj.get_data(ldtTimestamps, lsCheckStocks, "close")
""" Get data if needed """
if bPullVol:
dmVolume = norObj.get_data(ldtTimestamps, lsCheckStocks, "volume")
""" Only query this once if data is provided, else query every time with new data """
if lsValid is None or bPullVol or bPullPrice:
lsValid = stockFilter(dmPrice, dmVolume, fNonNan, fPriceVolume)
for sAdd in lsValid:
if sAdd in lsCheckStocks:
lsRetStocks.append(sAdd)
return sorted(lsRetStocks)
global dtStartTest
global dtEndTest
#reffered to john cornwell's code
if __name__ == '__main__':
lsSym = ['AA', 'AXP', 'BA', 'BAC', 'CAT', 'CSCO', 'CVX', 'DD', 'DIS', 'GE', 'HD', 'HPQ', 'IBM', 'INTC', 'JNJ', \
'JPM', 'KFT', 'KO', 'MCD', 'MMM', 'MRK', 'MSFT', 'PFE', 'PG', 'T', 'TRV', 'UTX', 'WMT', 'XOM','VZ' ]
dtStarttrain = dt.datetime(2007,01,01)
dtEndtrain = dt.datetime(2009,12,31)
dtStartTest = dt.datetime(2010,01,01)
dtEndTest = dt.datetime(2010,06,30)
norObj = da.DataAccess('Norgate')
lfdTimestamps = du.getNYSEdays(dtStarttrain, dtEndtrain, dt.timedelta(hours=16))
ldtTimestamps = du.getNYSEdays( dtStartTest, dtEndTest, dt.timedelta(hours=16) )
lTimestamps = du.getNYSEdays(dtStarttrain , dtEndTest , dt.timedelta(hours=16))
global dfPrice
global dfVolume
dfPrice = norObj.get_data( lTimestamps, lsSym, 'close' )
dfVolume = norObj.get_data( lTimestamps, lsSym, 'volume' )
lfcFeatures = [ featMA,featMA, featRSI,featRSI,featDrawDown,featRunUp,featVolumeDelta,featVolumeDelta,featAroon, classFutRet ]
global copyfeatures
copyfeatures = lfcFeatures
#ldArgs = [{}] * len(lfcFeatures)
''' Custom Arguments '''
ldArgs = [ {'lLookback':10, 'bRel':True},\
#
# Prepare to read the data
#
symbols = ["AAPL", "GLD", "GOOG", "SPY", "XOM"]
#
#for testing wth the graphs on wiki, uncomment the below two datetimes
#and comment the two lines following these two
#for actual submission, do the reverse
#
#startday = dt.datetime(2008,1,1)
#endday = dt.datetime(2009,12,31)
startday = dt.datetime(2007, 1, 1)
endday = dt.datetime(2010, 12, 31)
timeofday = dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday, endday, timeofday)
dataobj = da.DataAccess('Norgate')
voldata = dataobj.get_data(timestamps, symbols, "volume", verbose=True)
close = dataobj.get_data(timestamps, symbols, "close", verbose=True)
actualclose = dataobj.get_data(timestamps,
symbols,
"actual_close",
verbose=True)
#
# Plot the adjusted close data
#
plt.clf()
newtimestamps = close.index
pricedat = close.values # pull the 2D ndarray out of the pandas object
def getRandPort(lNum,
dtStart=None,
dtEnd=None,
lsStocks=None,
dmPrice=None,
dmVolume=None,
bFilter=True,
fNonNan=0.95,
fPriceVolume=100 * 1000,
lSeed=None):
"""
@summary Returns a random portfolio based on certain criteria.
@param lNum: Number of stocks to be included
@param dtStart: Start date for portfolio
@param dtEnd: End date for portfolio
@param lsStocks: Optional list of ticker symbols, if not provided all symbols will be used
@param bFilter: If False, stocks are not filtered by price or volume data, simply return random Portfolio.
@param dmPrice: Optional price data, if not provided, data access will be queried
@param dmVolume: Optional volume data, if not provided, data access will be queried
@param fNonNan: Optional non-nan percent for filter, default is .95
@param fPriceVolume: Optional price*volume for filter, default is 100,000
@warning: Does not work for all sets of optional inputs, e.g. if you don't include dtStart, dtEnd, you need
to include dmPrice/dmVolume
@return list of stocks which meet the criteria
"""
if (lsStocks is None):
if (dmPrice is None and dmVolume is None):
norObj = da.DataAccess('Norgate')
lsStocks = norObj.get_all_symbols()
elif (not dmPrice is None):
lsStocks = list(dmPrice.columns)
else:
lsStocks = list(dmVolume.columns)
if (dmPrice is None and dmVolume is None and bFilter == True):
norObj = da.DataAccess('Norgate')
ldtTimestamps = du.getNYSEdays(dtStart, dtEnd, dt.timedelta(hours=16))
''' if dmPrice and dmVol are provided then we don't query it every time '''
bPullPrice = False
bPullVol = False
if (dmPrice is None):
bPullPrice = True
if (dmVolume is None):
bPullVol = True
''' Default seed (none) uses system clock '''
rand.seed(lSeed)
lsRetStocks = []
''' Loop until we have enough randomly selected stocks '''
llRemainingIndexes = range(0, len(lsStocks))
lsValid = None
while (len(lsRetStocks) != lNum):
lsCheckStocks = []
for i in range(lNum - len(lsRetStocks)):
lRemaining = len(llRemainingIndexes)
if (lRemaining == 0):
print 'Error in getRandPort: ran out of stocks'
return lsRetStocks
''' Pick a stock and remove it from the list of remaining stocks '''
lPicked = rand.randint(0, lRemaining - 1)
lsCheckStocks.append(lsStocks[llRemainingIndexes.pop(lPicked)])
''' If bFilter is false, simply return our first list of stocks, don't check prive/vol '''
if (not bFilter):
return sorted(lsCheckStocks)
''' Get data if needed '''
if (bPullPrice):
dmPrice = norObj.get_data(ldtTimestamps, lsCheckStocks, 'close')
''' Get data if needed '''
if (bPullVol):
dmVolume = norObj.get_data(ldtTimestamps, lsCheckStocks, 'volume')
''' Only query this once if data is provided, else query every time with new data '''
if (lsValid is None or bPullVol or bPullPrice):
lsValid = stockFilter(dmPrice, dmVolume, fNonNan, fPriceVolume)
for sAdd in lsValid:
if sAdd in lsCheckStocks:
lsRetStocks.append(sAdd)
return sorted(lsRetStocks)
# Prepare to read the data
#
#
#Uncomment this to test with the picture on the wiki
#
#startday = dt.datetime(2010,1,1)
#endday = dt.datetime(2010,10,1)
#stock='VZ'
startday = dt.datetime(2009,1,1)
endday = dt.datetime(2010,1,1)
stock='IBM'
symbols = [stock]
timeofday=dt.timedelta(hours=16)
timestamps = du.getNYSEdays(startday,endday,timeofday)
dataobj = da.DataAccess('Norgate')
adjclose = dataobj.get_data(timestamps, symbols, "close")
adjclose = adjclose.fillna()
adjclose = adjclose.fillna(method='backfill')
# Get the 20 day moving avg and moving stddev
movavg = pa.rolling_mean(adjclose,20,min_periods=20)
movstddev = pa.rolling_std(adjclose, 20, min_periods=20)
# Compute the upper and lower bollinger bands
upperband = movavg + 2*movstddev
lowerband = movavg - 2*movstddev
plt.ylabel('Error')
#plt.show()
plt.savefig('FeatureTest.png', format='png')
if __name__ == '__main__':
''' Use Dow 30 '''
lsSym = ['AA', 'AXP', 'BA', 'BAC', 'CAT', 'CSCO', 'CVX', 'DD', 'DIS', 'GE', 'HD', 'HPQ', 'IBM', 'INTC', 'JNJ', \
'JPM', 'KFT', 'KO', 'MCD', 'MMM', 'MRK', 'MSFT', 'PFE', 'PG', 'T', 'TRV', 'UTX', 'WMT', 'XOM' ]
#lsSym = ['XOM']
''' Get data for 2009-2010 '''
dtStart = dt.datetime(2010, 8, 01)
dtEnd = dt.datetime(2010, 12, 31)
norObj = da.DataAccess('Norgate')
ldtTimestamps = du.getNYSEdays(dtStart, dtEnd, dt.timedelta(hours=16))
dfPrice = norObj.get_data(ldtTimestamps, lsSym, 'close')
dfVolume = norObj.get_data(ldtTimestamps, lsSym, 'volume')
''' Imported functions from qstkfeat.features, NOTE: last function is classification '''
lfcFeatures = [featMA, featRSI, classFutRet]
''' Default Arguments '''
#ldArgs = [{}] * len(lfcFeatures)
''' Custom Arguments '''
ldArgs = [ {'lLookback':30, 'bRel':True},\
{},\
{}]
''' Generate a list of DataFrames, one for each feature, with the same index/column structure as price data '''
ldfFeatures = ftu.applyFeatures(dfPrice, dfVolume, lfcFeatures, ldArgs)
bPlot = False
