def _prices(cls, local_dir, ticker,dud) :
filename = '{}/{}.pkl'.format(local_dir,ticker)
reader = EXTRACT.instance().reader
prices = reader.extract_from_yahoo(ticker)
if prices is None :
dud.append(ticker)
return dud
STOCK_TIMESERIES.save(filename, ticker, prices)
return dud
def save(cls, local_dir, ticker, dud=None):
if dud is None:
dud = []
data = cls.read(ticker)
if data is None:
dud.append(ticker)
return dud
filename = '{}/{}.pkl'.format(local_dir, ticker)
STOCK_TIMESERIES.save(filename, ticker, data)
return dud
def save(cls, local_dir, ticker, dud):
filename = '{}/{}.pkl'.format(local_dir, ticker)
if cls.reader is None:
cls.reader = STOCK_TIMESERIES.init()
prices = cls.reader.extract_from_yahoo(ticker)
if prices is None:
dud.append(ticker)
return dud
STOCK_TIMESERIES.save(filename, ticker, prices)
return dud
def _prices(cls, wait_on_failure, local_dir, ticker, dud):
if dud is None:
dud = []
filename = '{}/{}.pkl'.format(local_dir, ticker)
reader = STOCK_TIMESERIES.init()
prices = reader.extract_from_yahoo(ticker)
if prices is None:
dud.append(ticker)
time.sleep(wait_on_failure)
return dud
STOCK_TIMESERIES.save(filename, ticker, prices)
del prices
return dud
def read(file_list, stock_list):
for path in file_list:
name, ret = STOCK_TIMESERIES.load(path)
if name not in stock_list:
del ret
continue
yield name, ret
def read(cls, value_list):
repo = EXTRACT.instance().repo
ret = {}
for name, data in STOCK_TIMESERIES.read(repo, value_list):
logging.info((name, type(data), data.shape))
ret[name] = data
return ret
def load(file_list, value_list):
ret = {}
for name, data in STOCK_TIMESERIES.read(file_list, value_list):
if len(data) < 7 * FINANCE.YEAR:
logging.info(
"{} of length {} rejected for being less than {}".format(
name, len(data), 7 * FINANCE.YEAR))
continue
data = MONTECARLO.find(data['Adj Close'], period=FINANCE.YEAR)
#filter stocks that have less than a year
sharpe = data.get('sharpe', 0)
if sharpe == 0:
continue
#filter stocks that have negative returns
returns = data.get('returns', 0)
if returns <= 0:
logging.info(
"{} of returns {} rejected for being unprofitable".format(
name, returns))
continue
key_list = data.keys()
value_list = map(lambda x: data[x], key_list)
value_list = map(lambda x: round(x, 2), value_list)
msg = dict(zip(key_list, value_list))
logging.info((name, msg))
ret[name] = data
return ret
def read(cls, value_list) :
repo = VARIABLES().repo_stock
ret = {}
for name, data in STOCK_TIMESERIES.read(repo, value_list) :
logging.info((name,type(data),data.shape))
ret[name] = data
return ret
def prep(tickers):
reader = STOCK_TIMESERIES.init()
values = map(lambda ticker: reader.extract_from_yahoo(ticker), tickers)
values = map(lambda data: pd.DataFrame(data)['Adj Close'], values)
values = list(values)
ret = dict(zip(tickers, values))
ret = pd.DataFrame(ret, columns=tickers)
return ret
def main(local, *stock_list):
logging.info(stock_list)
reader = STOCK_TIMESERIES.init()
for stock in stock_list:
filename = '{}/historical_prices/{}.pkl'.format(local, stock)
ret = reader.extract_from_yahoo(stock)
logging.debug(ret.tail(5))
STOCK_TIMESERIES.save(filename, stock, ret)
if len(stock_list) > 0: return
nasdaq = NASDAQ.init()
for stock in nasdaq():
filename = '{}/historical_prices/{}.pkl'.format(local, stock)
ret = reader.extract_from_yahoo(stock)
STOCK_TIMESERIES.save(filename, stock, ret)
def action(stock_list):
target = 'file_list'
file_list = globals().get(target, [])
for path in file_list:
name, ret = STOCK_TIMESERIES.load(path)
if name not in stock_list:
del ret
continue
yield name, ret
def load(cls, data_store, ticker):
filename = '{}/{}.pkl'.format(data_store, ticker)
name, data = STOCK_TIMESERIES.load(filename)
if ticker == name:
return data
msg = '{} {}'.format(ticker, name)
msg = 'ticker does not match between filename and file content {}'.format(
msg)
raise ValueError(msg)
def prep(*file_list):
file_list = sorted(file_list)
spy_list = filter(lambda path: 'SPY' in path, file_list)
control = spy_list[0]
file_list = filter(lambda path: control != path, file_list)
label, data = STOCK_TIMESERIES.load(control)
print label
quarter = Partition.by_quarter(data)
return label, data, quarter, file_list
def loadData(*file_list):
file_list = sorted(file_list)
spy = filter(lambda stock: 'SPY' in stock, file_list)
spy_filename = spy[0]
spy_name, spy_data = STOCK_TIMESERIES.load(spy_filename)
spy_quarterly = StockTransform.by_quarter(spy_data)
key_list = spy_quarterly.keys()
file_list = filter(lambda path: spy_filename not in path, file_list)
for file_path in file_list:
name, data = STOCK_TIMESERIES.load(file_path)
quarterly = StockTransform.by_quarter(data)
for key in key_list:
if key not in quarterly: continue
if len(quarterly[key]) == 0: continue
yield key, name, data, quarterly[
key], spy_name, spy_data, spy_quarterly[key]
def load(file_list, stock_list) :
name_list = []
data_list = pd.DataFrame()
for name, stock in STOCK_TIMESERIES.read(file_list, stock_list) :
try :
data_list[name] = stock['Adj Close']
name_list.append(name)
except Exception as e : logging.error(e, exc_info=True)
finally : pass
return name_list, data_list
def _init(cls) :
if not (cls._data_list is None) :
return cls._data_list
target = 'test_finance_end'
end = globals().get(target,None)
reader = STOCK_TIMESERIES.init(end=end)
target = 'test_finance_stock_list'
stock_list = globals().get(target,[])
cls._data_list = map(lambda stock : reader.extract_from_yahoo(stock), stock_list)
cls._data_list = list(cls._data_list)
return cls._data_list
def load(value_list) :
file_list = PREP.singleton().file_list
ret = {}
for name, data in STOCK_TIMESERIES.read(file_list, value_list) :
flag, data = HELPER.is_stock_invalid(name, data)
if flag :
continue
ret[name] = data
msg = HELPER.round_values(**data)
logging.info((name, msg))
return ret
def prices(cls, ticker) :
data_store = cls.instance().file_list
suffix = '/{}.pkl'.format(ticker)
filename = filter(lambda x : x.endswith(suffix), data_store)
filename = list(filename)
if len(filename) > 0 :
filename = filename[0]
logging.info((filename,ticker))
name, ret = STOCK_TIMESERIES.load(filename)
ret = ret[cls._prices]
#ret = ret.reset_index(drop=False)
#ret.sort_index(inplace=True)
return ret
def prep():
target = 'file_list'
file_list = globals().get(target, [])
file_list = sorted(file_list)
spy_list = filter(lambda path: 'SPY' in path, file_list)
if not isinstance(spy_list, list):
spy_list = list(spy_list)
control = spy_list[0]
file_list = filter(lambda path: control != path, file_list)
if not isinstance(file_list, list):
file_list = list(file_list)
label, data = STOCK_TIMESERIES.load(control)
return label, data, file_list
def load(cls, data_store, ticker):
filename = '{}/{}.pkl'.format(data_store, ticker)
#if not os.path.exists(filename) :
# data = cls.read(ticker)
# STOCK_TIMESERIES.save(filename, ticker, data)
# return data
name, data = STOCK_TIMESERIES.load(filename)
if ticker == name:
return data
msg = '{} {}'.format(ticker, name)
msg = 'ticker does not match between filename and file content {}'.format(
msg)
raise ValueError(msg)
def process(file_list, value_list):
ret = {}
for name, data in STOCK_TIMESERIES.read(file_list, value_list):
data.sort_index(inplace=True)
data = data['Adj Close']
data = MONTECARLO.find(data,
risk_free_rate=0.02,
period=FINANCE.YEAR,
span=0)
# filter stocks that have less than a year
sharpe = data.get('sharpe', 0)
if sharpe == 0: continue
ret[name] = data
return ret
def load(self, *ticker_list):
filename_list = map(lambda ticker : '/{}.pkl'.format(ticker), ticker_list)
filename_list = list(filename_list)
logging.info((ticker_list,filename_list))
ret = {}
for i, suffix in enumerate(filename_list) :
filename = filter(lambda x : x.endswith(suffix), self.price_list)
filename = list(filename)
if len(filename) == 0 :
continue
filename = filename[0]
name, temp = STOCK_TIMESERIES.load(filename)
key = ticker_list[i]
ret[key] = temp[self.price_column]
return ret
def load(file_list, value_list):
ret = {}
for name, data in STOCK_TIMESERIES.read(file_list, value_list):
data = MONTECARLO.find(data['Adj Close'], period=FINANCE.YEAR)
#filter stocks that have less than a year
sharpe = data.get('sharpe', 0)
if sharpe == 0: continue
#filter stocks that have negative returns
returns = data.get('returns', 0)
if returns <= 0: continue
key_list = data.keys()
value_list = map(lambda x: data[x], key_list)
value_list = map(lambda x: round(x, 2), value_list)
msg = dict(zip(key_list, value_list))
logging.info((name, msg))
ret[name] = data
return ret
def instance(cls) :
if not (cls._singleton is None) :
return cls._singleton
target = 'env'
env = globals().get(target,None)
target = 'data_store_stock'
stock = globals().get(target,'')
if not isinstance(stock,str) :
stock = str(stock)
target = 'data_store_fund'
fund = globals().get(target,'')
if not isinstance(fund,str) :
fund = str(fund)
reader = STOCK_TIMESERIES.init()
env.mkdir(stock)
env.mkdir(fund)
cls._singleton = cls(env,stock,fund,reader)
return cls._singleton
def instance(cls, **kwargs) :
if not (cls._singleton is None) :
return cls._singleton
target = 'env'
_env = globals().get(target,None)
target = "data_store"
data_store = globals().get(target,[])
target = "output_file"
output_file = globals().get(target,[])
target = 'config_file'
config_file = globals().get(target,[])
if not isinstance(config_file,list) :
config_file = list(config_file)
if len(_env.argv) > 1 :
config_file = [_env.argv[1]]
if len(_env.argv) > 2 :
output_file = [_env.argv[2]]
reader = STOCK_TIMESERIES.init()
env.mkdir(data_store)
cls._singleton = cls(_env,data_store,config_file,output_file,reader)
return cls._singleton
def read(cls, ticker):
if cls.reader is None:
cls.reader = STOCK_TIMESERIES.init()
return cls.reader.extract_from_yahoo(ticker)
def load(cls, value_list):
ret = {}
for name, data in STOCK_TIMESERIES.read(cls.stock_data, value_list):
ret[name] = data['Adj Close']
return ret
def main(control_by_quarter, *file_list):
stable = {}
stable_key_list = [
'label', 'mean_volume', 'mean_vol_ratio', 'std_volume', 'std_vol_ratio'
]
performers = {}
performers_key_list = [
'label', 'mean_adj_close', 'mean_close_ratio', 'dev_adj_close',
'std_close_ratio'
]
ini_stable = {}
ini_performance = {}
for test in file_list:
test_label, test = STOCK_TIMESERIES.load(test)
if len(test) < 100:
del test
continue
test = Partition.by_quarter(test)
for q in sorted(test.keys()):
test_vol = test[q]['Volume']
control_vol = control_by_quarter[q]['Volume']
comp_mean = round(test_vol.mean() / control_vol.mean(), 2)
comp_std = round(test_vol.std() / control_vol.std(), 2)
if comp_std > 1.5:
continue
if test_label not in stable.keys(): stable[test_label] = {}
label = '{} vs {}'.format(control_label, test_label)
value_list = [
label,
round(test_vol.mean(), 2), comp_mean,
round(test_vol.std(), 2), comp_std
]
stable[test_label][q] = dict(zip(stable_key_list, value_list))
test_close = test[q]['Adj Close']
control_close = control_by_quarter[q]['Adj Close']
comp_mean = round(test_close.mean() / control_close.mean(), 2)
comp_std = round(test_close.std() / control_close.std(), 2)
if comp_mean < 1:
continue
label = '{} vs {}'.format(control_label, test_label)
value_list = [
label,
round(test_close.mean(), 2), comp_mean,
round(test_close.std(), 2), comp_std
]
if test_label not in performers.keys(): performers[test_label] = {}
performers[test_label][q] = dict(
zip(performers_key_list, value_list))
if test_label in stable.keys():
key_list = sorted(stable[test_label].keys())
key_list = map(lambda x: str(x), key_list)
key_list = "".join(key_list)
key = 'Q{}'.format(key_list)
if key not in ini_stable:
ini_stable[key] = []
print "Safe ", test_label, key, stable[test_label]
ini_stable[key].append(test_label)
if test_label in performers.keys():
key_list = sorted(performers[test_label].keys())
key_list = map(lambda x: str(x), key_list)
key_list = "".join(key_list)
key = 'Q{}'.format(key_list)
if key not in ini_performance:
ini_performance[key] = []
print "Good ", test_label, key, performers[test_label]
ini_performance[key].append(test_label)
del test
return ini_stable, ini_performance
def load(file_list, stock_list) :
name_list,data_list = STOCK_TIMESERIES.read_all(file_list, stock_list)
data_list = STOCK_TIMESERIES.flatten('Adj Close',data_list)
data_list = data_list.fillna(0)
logging.debug(data_list)
return name_list, data_list
log_msg = '%(module)s.%(funcName)s(%(lineno)s) %(levelname)s - %(message)s'
logging.basicConfig(stream=sys.stdout, format=log_msg, level=logging.DEBUG)
def prep(*ini_list) :
ini_list = filter(lambda x : "benchmark" in x , ini_list)
print (ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list) :
if section == 'Index' : pass
else : continue
yield key, stock_list
file_list = env.list_filenames('local/historical_prices/*pkl')
ini_list = env.list_filenames('local/*.ini')
reader = STOCK_TIMESERIES.init()
for name, stock_list in prep(*ini_list) :
for stock in stock_list :
print ((stock,name))
data = reader.extract_from_yahoo(stock)
if data is None : continue
ret = data[['Adj Close']]
print (ret.head(2))
print (ret.tail(2))
print (ret.mean())
print (ret.std())
print (ret.mean()[0])
print (ret.std()[0])
print (HELPER.find(ret,period=FINANCE.YEAR,span=0))
print (HELPER.find(ret,period=FINANCE.YEAR))
print ((stock,name))
