def _prices(cls, local_dir, ticker,dud) : filename = '{}/{}.pkl'.format(local_dir,ticker) reader = EXTRACT.instance().reader prices = reader.extract_from_yahoo(ticker) if prices is None : dud.append(ticker) return dud STOCK_TIMESERIES.save(filename, ticker, prices) return dud
def save(cls, local_dir, ticker, dud=None): if dud is None: dud = [] data = cls.read(ticker) if data is None: dud.append(ticker) return dud filename = '{}/{}.pkl'.format(local_dir, ticker) STOCK_TIMESERIES.save(filename, ticker, data) return dud
def save(cls, local_dir, ticker, dud): filename = '{}/{}.pkl'.format(local_dir, ticker) if cls.reader is None: cls.reader = STOCK_TIMESERIES.init() prices = cls.reader.extract_from_yahoo(ticker) if prices is None: dud.append(ticker) return dud STOCK_TIMESERIES.save(filename, ticker, prices) return dud
def _prices(cls, wait_on_failure, local_dir, ticker, dud): if dud is None: dud = [] filename = '{}/{}.pkl'.format(local_dir, ticker) reader = STOCK_TIMESERIES.init() prices = reader.extract_from_yahoo(ticker) if prices is None: dud.append(ticker) time.sleep(wait_on_failure) return dud STOCK_TIMESERIES.save(filename, ticker, prices) del prices return dud
def read(file_list, stock_list): for path in file_list: name, ret = STOCK_TIMESERIES.load(path) if name not in stock_list: del ret continue yield name, ret
def read(cls, value_list): repo = EXTRACT.instance().repo ret = {} for name, data in STOCK_TIMESERIES.read(repo, value_list): logging.info((name, type(data), data.shape)) ret[name] = data return ret
def load(file_list, value_list): ret = {} for name, data in STOCK_TIMESERIES.read(file_list, value_list): if len(data) < 7 * FINANCE.YEAR: logging.info( "{} of length {} rejected for being less than {}".format( name, len(data), 7 * FINANCE.YEAR)) continue data = MONTECARLO.find(data['Adj Close'], period=FINANCE.YEAR) #filter stocks that have less than a year sharpe = data.get('sharpe', 0) if sharpe == 0: continue #filter stocks that have negative returns returns = data.get('returns', 0) if returns <= 0: logging.info( "{} of returns {} rejected for being unprofitable".format( name, returns)) continue key_list = data.keys() value_list = map(lambda x: data[x], key_list) value_list = map(lambda x: round(x, 2), value_list) msg = dict(zip(key_list, value_list)) logging.info((name, msg)) ret[name] = data return ret
def read(cls, value_list) : repo = VARIABLES().repo_stock ret = {} for name, data in STOCK_TIMESERIES.read(repo, value_list) : logging.info((name,type(data),data.shape)) ret[name] = data return ret
def prep(tickers): reader = STOCK_TIMESERIES.init() values = map(lambda ticker: reader.extract_from_yahoo(ticker), tickers) values = map(lambda data: pd.DataFrame(data)['Adj Close'], values) values = list(values) ret = dict(zip(tickers, values)) ret = pd.DataFrame(ret, columns=tickers) return ret
def main(local, *stock_list): logging.info(stock_list) reader = STOCK_TIMESERIES.init() for stock in stock_list: filename = '{}/historical_prices/{}.pkl'.format(local, stock) ret = reader.extract_from_yahoo(stock) logging.debug(ret.tail(5)) STOCK_TIMESERIES.save(filename, stock, ret) if len(stock_list) > 0: return nasdaq = NASDAQ.init() for stock in nasdaq(): filename = '{}/historical_prices/{}.pkl'.format(local, stock) ret = reader.extract_from_yahoo(stock) STOCK_TIMESERIES.save(filename, stock, ret)
def action(stock_list): target = 'file_list' file_list = globals().get(target, []) for path in file_list: name, ret = STOCK_TIMESERIES.load(path) if name not in stock_list: del ret continue yield name, ret
def load(cls, data_store, ticker): filename = '{}/{}.pkl'.format(data_store, ticker) name, data = STOCK_TIMESERIES.load(filename) if ticker == name: return data msg = '{} {}'.format(ticker, name) msg = 'ticker does not match between filename and file content {}'.format( msg) raise ValueError(msg)
def prep(*file_list): file_list = sorted(file_list) spy_list = filter(lambda path: 'SPY' in path, file_list) control = spy_list[0] file_list = filter(lambda path: control != path, file_list) label, data = STOCK_TIMESERIES.load(control) print label quarter = Partition.by_quarter(data) return label, data, quarter, file_list
def loadData(*file_list): file_list = sorted(file_list) spy = filter(lambda stock: 'SPY' in stock, file_list) spy_filename = spy[0] spy_name, spy_data = STOCK_TIMESERIES.load(spy_filename) spy_quarterly = StockTransform.by_quarter(spy_data) key_list = spy_quarterly.keys() file_list = filter(lambda path: spy_filename not in path, file_list) for file_path in file_list: name, data = STOCK_TIMESERIES.load(file_path) quarterly = StockTransform.by_quarter(data) for key in key_list: if key not in quarterly: continue if len(quarterly[key]) == 0: continue yield key, name, data, quarterly[ key], spy_name, spy_data, spy_quarterly[key]
def load(file_list, stock_list) : name_list = [] data_list = pd.DataFrame() for name, stock in STOCK_TIMESERIES.read(file_list, stock_list) : try : data_list[name] = stock['Adj Close'] name_list.append(name) except Exception as e : logging.error(e, exc_info=True) finally : pass return name_list, data_list
def _init(cls) : if not (cls._data_list is None) : return cls._data_list target = 'test_finance_end' end = globals().get(target,None) reader = STOCK_TIMESERIES.init(end=end) target = 'test_finance_stock_list' stock_list = globals().get(target,[]) cls._data_list = map(lambda stock : reader.extract_from_yahoo(stock), stock_list) cls._data_list = list(cls._data_list) return cls._data_list
def load(value_list) : file_list = PREP.singleton().file_list ret = {} for name, data in STOCK_TIMESERIES.read(file_list, value_list) : flag, data = HELPER.is_stock_invalid(name, data) if flag : continue ret[name] = data msg = HELPER.round_values(**data) logging.info((name, msg)) return ret
def prices(cls, ticker) : data_store = cls.instance().file_list suffix = '/{}.pkl'.format(ticker) filename = filter(lambda x : x.endswith(suffix), data_store) filename = list(filename) if len(filename) > 0 : filename = filename[0] logging.info((filename,ticker)) name, ret = STOCK_TIMESERIES.load(filename) ret = ret[cls._prices] #ret = ret.reset_index(drop=False) #ret.sort_index(inplace=True) return ret
def prep(): target = 'file_list' file_list = globals().get(target, []) file_list = sorted(file_list) spy_list = filter(lambda path: 'SPY' in path, file_list) if not isinstance(spy_list, list): spy_list = list(spy_list) control = spy_list[0] file_list = filter(lambda path: control != path, file_list) if not isinstance(file_list, list): file_list = list(file_list) label, data = STOCK_TIMESERIES.load(control) return label, data, file_list
def load(cls, data_store, ticker): filename = '{}/{}.pkl'.format(data_store, ticker) #if not os.path.exists(filename) : # data = cls.read(ticker) # STOCK_TIMESERIES.save(filename, ticker, data) # return data name, data = STOCK_TIMESERIES.load(filename) if ticker == name: return data msg = '{} {}'.format(ticker, name) msg = 'ticker does not match between filename and file content {}'.format( msg) raise ValueError(msg)
def process(file_list, value_list): ret = {} for name, data in STOCK_TIMESERIES.read(file_list, value_list): data.sort_index(inplace=True) data = data['Adj Close'] data = MONTECARLO.find(data, risk_free_rate=0.02, period=FINANCE.YEAR, span=0) # filter stocks that have less than a year sharpe = data.get('sharpe', 0) if sharpe == 0: continue ret[name] = data return ret
def load(self, *ticker_list): filename_list = map(lambda ticker : '/{}.pkl'.format(ticker), ticker_list) filename_list = list(filename_list) logging.info((ticker_list,filename_list)) ret = {} for i, suffix in enumerate(filename_list) : filename = filter(lambda x : x.endswith(suffix), self.price_list) filename = list(filename) if len(filename) == 0 : continue filename = filename[0] name, temp = STOCK_TIMESERIES.load(filename) key = ticker_list[i] ret[key] = temp[self.price_column] return ret
def load(file_list, value_list): ret = {} for name, data in STOCK_TIMESERIES.read(file_list, value_list): data = MONTECARLO.find(data['Adj Close'], period=FINANCE.YEAR) #filter stocks that have less than a year sharpe = data.get('sharpe', 0) if sharpe == 0: continue #filter stocks that have negative returns returns = data.get('returns', 0) if returns <= 0: continue key_list = data.keys() value_list = map(lambda x: data[x], key_list) value_list = map(lambda x: round(x, 2), value_list) msg = dict(zip(key_list, value_list)) logging.info((name, msg)) ret[name] = data return ret
def instance(cls) : if not (cls._singleton is None) : return cls._singleton target = 'env' env = globals().get(target,None) target = 'data_store_stock' stock = globals().get(target,'') if not isinstance(stock,str) : stock = str(stock) target = 'data_store_fund' fund = globals().get(target,'') if not isinstance(fund,str) : fund = str(fund) reader = STOCK_TIMESERIES.init() env.mkdir(stock) env.mkdir(fund) cls._singleton = cls(env,stock,fund,reader) return cls._singleton
def instance(cls, **kwargs) : if not (cls._singleton is None) : return cls._singleton target = 'env' _env = globals().get(target,None) target = "data_store" data_store = globals().get(target,[]) target = "output_file" output_file = globals().get(target,[]) target = 'config_file' config_file = globals().get(target,[]) if not isinstance(config_file,list) : config_file = list(config_file) if len(_env.argv) > 1 : config_file = [_env.argv[1]] if len(_env.argv) > 2 : output_file = [_env.argv[2]] reader = STOCK_TIMESERIES.init() env.mkdir(data_store) cls._singleton = cls(_env,data_store,config_file,output_file,reader) return cls._singleton
def read(cls, ticker): if cls.reader is None: cls.reader = STOCK_TIMESERIES.init() return cls.reader.extract_from_yahoo(ticker)
def load(cls, value_list): ret = {} for name, data in STOCK_TIMESERIES.read(cls.stock_data, value_list): ret[name] = data['Adj Close'] return ret
def main(control_by_quarter, *file_list): stable = {} stable_key_list = [ 'label', 'mean_volume', 'mean_vol_ratio', 'std_volume', 'std_vol_ratio' ] performers = {} performers_key_list = [ 'label', 'mean_adj_close', 'mean_close_ratio', 'dev_adj_close', 'std_close_ratio' ] ini_stable = {} ini_performance = {} for test in file_list: test_label, test = STOCK_TIMESERIES.load(test) if len(test) < 100: del test continue test = Partition.by_quarter(test) for q in sorted(test.keys()): test_vol = test[q]['Volume'] control_vol = control_by_quarter[q]['Volume'] comp_mean = round(test_vol.mean() / control_vol.mean(), 2) comp_std = round(test_vol.std() / control_vol.std(), 2) if comp_std > 1.5: continue if test_label not in stable.keys(): stable[test_label] = {} label = '{} vs {}'.format(control_label, test_label) value_list = [ label, round(test_vol.mean(), 2), comp_mean, round(test_vol.std(), 2), comp_std ] stable[test_label][q] = dict(zip(stable_key_list, value_list)) test_close = test[q]['Adj Close'] control_close = control_by_quarter[q]['Adj Close'] comp_mean = round(test_close.mean() / control_close.mean(), 2) comp_std = round(test_close.std() / control_close.std(), 2) if comp_mean < 1: continue label = '{} vs {}'.format(control_label, test_label) value_list = [ label, round(test_close.mean(), 2), comp_mean, round(test_close.std(), 2), comp_std ] if test_label not in performers.keys(): performers[test_label] = {} performers[test_label][q] = dict( zip(performers_key_list, value_list)) if test_label in stable.keys(): key_list = sorted(stable[test_label].keys()) key_list = map(lambda x: str(x), key_list) key_list = "".join(key_list) key = 'Q{}'.format(key_list) if key not in ini_stable: ini_stable[key] = [] print "Safe ", test_label, key, stable[test_label] ini_stable[key].append(test_label) if test_label in performers.keys(): key_list = sorted(performers[test_label].keys()) key_list = map(lambda x: str(x), key_list) key_list = "".join(key_list) key = 'Q{}'.format(key_list) if key not in ini_performance: ini_performance[key] = [] print "Good ", test_label, key, performers[test_label] ini_performance[key].append(test_label) del test return ini_stable, ini_performance
def load(file_list, stock_list) : name_list,data_list = STOCK_TIMESERIES.read_all(file_list, stock_list) data_list = STOCK_TIMESERIES.flatten('Adj Close',data_list) data_list = data_list.fillna(0) logging.debug(data_list) return name_list, data_list
log_msg = '%(module)s.%(funcName)s(%(lineno)s) %(levelname)s - %(message)s' logging.basicConfig(stream=sys.stdout, format=log_msg, level=logging.DEBUG) def prep(*ini_list) : ini_list = filter(lambda x : "benchmark" in x , ini_list) print (ini_list) for path, section, key, stock_list in INI.loadList(*ini_list) : if section == 'Index' : pass else : continue yield key, stock_list file_list = env.list_filenames('local/historical_prices/*pkl') ini_list = env.list_filenames('local/*.ini') reader = STOCK_TIMESERIES.init() for name, stock_list in prep(*ini_list) : for stock in stock_list : print ((stock,name)) data = reader.extract_from_yahoo(stock) if data is None : continue ret = data[['Adj Close']] print (ret.head(2)) print (ret.tail(2)) print (ret.mean()) print (ret.std()) print (ret.mean()[0]) print (ret.std()[0]) print (HELPER.find(ret,period=FINANCE.YEAR,span=0)) print (HELPER.find(ret,period=FINANCE.YEAR)) print ((stock,name))