def main(save_file) :
ret = INI.init()
for key, value in action() :
logging.debug(value)
INI.write_section(ret,key,**value)
ret.write(open(save_file, 'w'))
logging.info("results saved to {}".format(save_file))
def config(cls, save_file, **config) :
logging.info("results saved to {}".format(save_file))
ret = INI.init()
for key in sorted(config) :
value = config.get(key,{})
INI.write_section(ret,key,**value)
ret.write(open(save_file, 'w'))
def main(file_list, ini_list, save_file):
logging.info("loading results {}".format(ini_list))
ret = INI.init()
for key, value in action(file_list, ini_list):
logging.debug(value)
INI.write_section(ret, key, **value)
ret.write(open(save_file, 'w'))
logging.info("results saved to {}".format(save_file))
def background(cls, **config):
save_file = EXTRACT.instance().background_file
ret = INI.init()
for key in sorted(config):
value = config.get(key, [])
INI.write_section(ret, key, **value)
ret.write(open(save_file, 'w'))
logging.info("results saved to {}".format(save_file))
def config(cls, data):
save_file = EXTRACT.singleton().output_file
ret = INI.init()
for key in sorted(data):
logging.info(key)
value = data[key]
INI.write_section(ret, key, **value)
ret.write(open(save_file, 'w'))
logging.info('writing to file {}'.format(save_file))
def main():
env = globals().get('env', None)
nasdaq = '{}/local/{}'.format(env.pwd_parent, NASDAQ.path)
finder = NASDAQ.init(filename=nasdaq)
stock_list = map(lambda stock: PROFILE.get(stock), finder())
ini, empty_list = find(stock_list)
stock_ini = '{pwd_parent}/local/yahoo_background.ini'.format(**vars(env))
config = INI.init()
temp = ini['Background']
for key in sorted(temp.keys()):
INI.write_section(config, key, **temp[key])
config.write(open(stock_ini, 'w'))
stock_ini = '{pwd_parent}/local/yahoo_background_sector.ini'.format(
**vars(env))
config = INI.init()
temp = ini['SectorGroupBy']
for key in sorted(temp.keys()):
INI.write_section(config, key, **temp[key])
config.write(open(stock_ini, 'w'))
# At one time, industries contained different sectors, this is no longer the case
stock_ini = '{pwd_parent}/local/yahoo_background_industry.ini'.format(
**vars(env))
config = INI.init()
temp = ini['IndustryGroupBy']
for key in sorted(temp.keys()):
INI.write_section(config, key, **temp[key])
config.write(open(stock_ini, 'w'))
return empty_list
def config(cls, summary, portfolio):
save_file = EXTRACT.instance().output_file
ret = INI.init()
INI.write_section(ret, "summary", **summary)
for key in portfolio.keys():
values = portfolio[key]
if not isinstance(values, dict):
values = values.to_dict()
INI.write_section(ret, key, **values)
ret.write(open(save_file, 'w'))
logging.info('saving file {}'.format(save_file))
def portfolio(cls, save_file, **portfolio) :
logging.info("saving results to file {}".format(save_file))
ret = INI.init()
name_list = sorted(portfolio.keys())
value_list = map(lambda key : portfolio[key], name_list)
for i, name in enumerate(name_list) :
if not isinstance(value_list,list) :
value_list = list(value_list)
INI.write_section(ret,name,**value_list[i])
ret.write(open(save_file, 'w'))
logging.info("results saved to file {}".format(save_file))
def main(local_dir) :
for name, section_list in action() :
output_file = "{}/portfolio_{}.ini".format(local_dir, name)
output_file = output_file.replace(" ", "_")
ret = INI.init()
name_list = sorted(section_list.keys())
value_list = map(lambda key : section_list[key], name_list)
for i, name in enumerate(name_list) :
if not isinstance(value_list,list) :
value_list = list(value_list)
INI.write_section(ret,name,**value_list[i])
logging.info("saving results to file {}".format(output_file))
ret.write(open(output_file, 'w'))
def draft(cls,data) :
save_file = EXTRACT.instance().draft
logging.info('Loading results : {}'.format(save_file))
config = INI.init()
for i, SECTION in enumerate(sorted(data)) :
target = 'alias'
if SECTION == target :
alias = data.get(SECTION)
continue
logging.info((i,SECTION))
value = data.get(SECTION)
INI.write_section(config, SECTION, **value)
for name in sorted(alias) :
INI.write_section(config,name,**alias[name])
config.write(open(save_file, 'w'))
def prep(*ini_list):
ret = {}
for path, section, key, weight in INI.loadList(*ini_list):
if section not in ret:
ret[section] = {}
ret[section][key] = float(weight[0])
return ret
def prep(*ini_list):
target_sector = 'target_sector'
target_sector = globals().get(target_sector, '_')
risk = {}
sharpe = {}
stock_list = {}
for path, section, key, value in INI.loadList(*ini_list):
if target_sector in key: pass
else: continue
if section not in risk:
risk[section] = {}
sharpe[section] = {}
if key.endswith(target_sector):
key = key.replace(section, '')
key = key.replace(target_sector, '')
stock_list[section] = value
else:
key = key.replace(section, '')
key = key.replace(target_sector, '')
if 'risk' in key:
key = key.replace('risk_', '')
key = key.replace('_', '')
risk[section][key] = value[0]
if 'sharpe' in key:
key = key.replace('sharpe_', '')
key = key.replace('_', '')
sharpe[section][key] = value[0]
for key in risk:
yield key, stock_list[key], risk[key], sharpe[key]
def prep(*ini_list) :
ini_list = filter(lambda x : "benchmark" in x , ini_list)
print (ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list) :
if section == 'Index' : pass
else : continue
yield key, stock_list
def filterByNasdaq(*ini_list):
performers = {}
stability = {}
Sector = {}
Industry = {}
FundFamily = {}
Category = {}
ini_list = filter(lambda file: "nasdaq_quarterly.ini" in file, ini_list)
for file_name, section, key, value in INI.loadList(*ini_list):
if section == "Stability":
config = stability
elif section == "Performance":
config = performers
elif section == "Sector":
config = Sector
elif section == "Industry":
config = Industry
elif section == "Category":
config = Category
elif section == "FundFamily":
config = FundFamily
else:
continue
config[key] = value
ret = performers.get('Q1234', [])
stock_list = _combineNasdaq(Sector, Industry)
fund_list = _combineNasdaq(Category, FundFamily)
ret_stock = sorted(list(set(ret) & set(stock_list)))
ret_fund = sorted(list(set(ret) & set(fund_list)))
return stock_list, fund_list, ret_stock, ret_fund
def config(cls):
ini_list = cls.instance().input_file
logging.info('Reading input file {}'.format(ini_list))
ret = {}
for path, section, key, value_list in INI.loadList(*[ini_list]):
if section not in ret:
ret[section] = {}
ret[section][key] = value_list
return ret
def background(cls):
load_file = cls.instance().background
if not (cls._background_cache is None):
logging.info('reading cache {}'.format(load_file))
return cls._background_cache
logging.info('reading file {}'.format(load_file))
ret = {}
for path, key, stock, value in INI.loadList(*load_file):
if "File Creation Time" in stock:
continue
if stock not in ret:
ret[stock] = {}
if key in cls._floats_in_summary:
value = float(value[0])
else:
value = ', '.join(value)
ret[stock][key] = value
load_file = cls.instance().sector
logging.info('reading file {}'.format(load_file))
for path, section, key, ticker_list in INI.loadList(*load_file):
entity = 'stock'
if 'fund' in path:
key = '{} ({})'.format(section, key)
entity = 'fund'
for ticker in ticker_list:
if ticker not in ret:
ret[ticker] = {}
ret[ticker]['SECTOR'] = key
ret[ticker]['ENTITY'] = entity
ret = pd.DataFrame(ret).T
ret['SECTOR'] = ret['SECTOR'].fillna("Unknown")
ret['NAME'] = ret['NAME'].fillna("Unavailable")
ret.fillna(0.0, inplace=True)
for sector, category in TRANSFORM.by_sector(ret):
pass
ret = cls.filterBackground(ret)
for sector, category in TRANSFORM.by_sector(ret):
pass
cls._background_cache = ret
return ret
def readPortfolio(cls):
portfolio = EXTRACT.instance().input_file
logging.info('reading file {}'.format(portfolio))
ret = {}
for path, section, key, weight in INI.loadList(*[portfolio]):
if section.startswith('dep_'):
continue
if section not in ret:
ret[section] = {}
ret[section][key] = float(weight[0])
return ret
def enrich(*ini_list):
ret = {}
ini_list = filter(lambda x: 'background' in x, ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list):
if section == 'Sector': pass
elif section == 'Industry': pass
else: continue
for stock in stock_list:
if stock not in ret: ret[stock] = {}
ret[stock][section] = key
return ret
def read(cls):
data = cls.singleton().input_file
logging.info('reading file {}'.format([data]))
ret = {}
for path, section, stock_sector, stock_list in INI.loadList(*[data]):
if section not in ret:
ret[section] = {}
ret[section][stock_sector] = stock_list
target = 'MERGED'
ret = ret.get(target, ret)
stock_list = cls.flatten(ret.values())
return ret, stock_list
def init(*ini_list):
performers = {}
stability = {}
for file_name, name, key, value in INI.loadList(*ini_list):
config = None
if name == "Stability":
config = stability
if name == "Performance":
config = performers
config[key] = value
ret = performers.get('Q1234', [])
return ret
def benchmark(*ini_list):
ret = {}
ini_list = filter(lambda x: 'benchmark' in x, ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list):
if section not in ['MOTLEYFOOL', 'Index']: continue
if section == 'MOTLEYFOOL':
if 'NASDAQ' not in key: continue
if 'FUND' not in key: continue
if section == 'Index':
if '500' not in key: continue
ret[key] = stock_list
return ret
def merge(cls) :
config = EXTRACT.instance().draft
logging.info('loading config files {}'.format(config))
ret = DICT_HELPER.init()
for path, section, key, stock in INI.loadList(*[config]) :
ret.append(key,*stock)
omit_list = ['ACT Symbol', 'CQS Symbol', 'alias', 'unknown']
for omit in omit_list :
ret.data.pop(omit,None)
stock_list = ret.values()
return ret.data, stock_list
def prep(*ini_list):
ret, overflow, default = prep_init()
for path, section, key, value in INI.loadList(*ini_list):
curr = lambdaFindSector(overflow, ret, section)
curr[section][key] = float(value[0])
default_keys = sorted(default)
for key in sorted(ret):
portfolio_list = ret[key]
portfolio_keys = sorted(portfolio_list)
_list = []
for portfolio_name in portfolio_keys:
portfolio = portfolio_list[portfolio_name]
value = map(lambda x: portfolio[x], default_keys)
_list.append(dict(zip(default_keys, value)))
yield key, dict(zip(portfolio_keys, _list))
def read(cls):
benchmark = EXTRACT.instance()._benchmark
logging.info('reading file {}'.format(benchmark))
ret = {}
for path, section, key, stock_list in INI.loadList(*benchmark):
#if section not in ['MOTLEYFOOL', 'Index'] : continue
if section not in ['PERSONAL', 'Index']: continue
if section == 'MOTLEYFOOL':
if 'NASDAQ' not in key: continue
if 'FUND' not in key: continue
if section == 'Index':
if '500' not in key: continue
ret[key] = stock_list
logging.debug(ret)
return ret
def _readSector(cls):
load_file = EXTRACT.instance().category
if not (cls._cache is None):
logging.info('reading cache {}'.format(load_file))
return cls._cache
logging.info('reading file {}'.format(load_file))
ret = {}
for path, section, key, ticker_list in INI.loadList(*load_file):
if 'fund' in path:
key = '{} ({})'.format(section, key)
for ticker in ticker_list:
if ticker not in ret:
ret[ticker] = {}
ret[ticker]['Sector'] = key
cls._cache = ret
return ret
def _prep(*ini_list) :
ret = {}
target = "stocks"
for path, section, key, value in INI.loadList(*ini_list) :
key = key.replace(section+"_","")
ret[key] = value
if target not in key :
continue
logging.debug(key)
prefix = key.replace(target,"")
ret = HELPER2.transform(prefix,ret)
if HELPER2.sharpe_cap(ret) :
ret = {}
continue
logging.info((section,ret))
yield section, ret.get(target,[])
ret = {}
def prep(target, *ini_list) :
Stock = {}
Fund = {}
ini_list = filter(lambda x : "sharpe_nasdaq" in x , ini_list)
ini_list = filter(lambda x : "fact" not in x , ini_list)
ini_list = filter(lambda x : target in x , ini_list)
logging.debug(ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list) :
if section == 'Sector' : config = Stock
elif section == 'Industry' : config = Stock
elif section == 'Fund' : config = Fund
else : continue
config[key] = stock_list
Stock = reduce(lambda a, b : a + b, Stock.values())
Stock = list(set(Stock))
Fund = reduce(lambda a, b : a + b, Fund.values())
Fund = list(set(Fund))
return sorted(Stock), sorted(Fund)
def prep(target, *ini_list):
Stock = {}
Fund = {}
ini_list = filter(lambda x: target in x, ini_list)
for path, section, key, stock_list in INI.loadList(*ini_list):
if section == 'Sector': config = Stock
elif section == 'Industry': config = Stock
elif section == 'Fund': config = Fund
else: continue
config[key] = stock_list
stock_list = Stock.values()
if len(stock_list) > 0:
Stock = reduce(lambda a, b: a + b, stock_list)
Stock = list(set(Stock))
fund_list = Fund.values()
if len(fund_list) > 0:
Fund = reduce(lambda a, b: a + b, fund_list)
Fund = list(set(Fund))
return sorted(Stock), sorted(Fund)
def read(cls):
load_file = EXTRACT.instance().category
if not (cls._cache is None):
return cls._cache
logging.info('reading file {}'.format(load_file))
ret = {}
for path, section, key, ticker_list in INI.loadList(*load_file):
entity = 'Stock'
if 'fund' in path:
key = '{} ({})'.format(section, key)
entity = 'Fund'
for ticker in ticker_list:
if ticker not in ret:
ret[ticker] = {}
ret[ticker]['SECTOR'] = key
ret[ticker]['ENTITY'] = entity
ret = pd.DataFrame(ret).T
logging.info(ret)
cls._cache = ret
return ret
def getByNasdaq(*ini_list):
Sector = {}
Industry = {}
FundFamily = {}
Category = {}
ini_list = filter(lambda file: "yahoo" in file, ini_list)
ini_list = filter(lambda file: "background" in file, ini_list)
logging.debug(ini_list)
for file_name, section, key, value in INI.loadList(*ini_list):
if section == "Sector":
config = Sector
elif section == "Industry":
config = Industry
elif section == "Category":
config = Category
elif section == "FundFamily":
config = FundFamily
else:
continue
config[key] = value
return Sector, Industry, Category, FundFamily