def read_tickers_from_file_or_web(stock_data_container_file, reload_file=False, dict_with_stock_pages_to_read={}):
"""
TODO
read the sp500 and CDAX tickers and saves it to given file
:param dict_with_stock_pages_to_read:
:param stock_data_container_file:
:param stock_exchange_file:
:param names_file:
:param reload_file: reload the tickers
:param tickers_file: file to save the tickers
:return: stock_data_container_list
"""
# TODO:
# https://de.wikipedia.org/wiki/Liste_von_Aktienindizes
# https://de.wikipedia.org/wiki/EURO_STOXX_50#Zusammensetzung
stock_data_container_list = []
if not os.path.exists(stock_data_container_file) or reload_file:
logger.info("Start reading tickers...")
pool = CommonUtils.get_threading_pool()
list_w = dict_with_stock_pages_to_read.values()
result_list = pool.map(CommonUtils.read_table_columns_from_webpage_list, list_w)
for result in result_list:
stock_data_container_list.extend(result)
# TODO: b) General Standard is not included of page:
# http://topforeignstocks.com/stock-lists/the-list-of-listed-companies-in-germany/
# TODO temp disabled: wartung
# TODO: http://www.boerse-online.de/index/liste/cdax
# no tickers symbols available, column 2 contains security (=name)
# all_names += read_table_column_from_wikipedia(
# 'https://de.wikipedia.org/wiki/Liste_der_im_CDAX_gelisteten_Aktien',
# 'wikitable sortable zebra', 2)
# from DataRead_Google_Yahoo import __get_symbols_from_names
# all_exchanges = []
# all_exchanges += list(repeat("de", len(all_names)))
# tickers, names_with_symbols = __get_symbols_from_names (all_names, all_exchanges)
# stock_tickers_names['tickers'] += tickers
# stock_tickers_names['_names'] += names_with_symbols
# stock_tickers_names['_stock_exchange'] += list(repeat("de", len(names_with_symbols)))
with open(stock_data_container_file, "wb") as f:
pickle.dump(stock_data_container_list, f)
else:
with open(stock_data_container_file, "rb") as f:
stock_data_container_list += pickle.load(f)
return stock_data_container_list
def read_tickers_from_web(stock_data_container_file, dict_with_stock_pages_to_read={}):
"""
Read the gives list of stock pages
:param dict_with_stock_pages_to_read:
:param stock_data_container_file:
:return: stock_data_container_list
"""
stock_data_container_list = []
logger.info("Start reading tickers from web...")
pool = CommonUtils.get_threading_pool()
list_w = dict_with_stock_pages_to_read.values()
result_list = pool.map(CommonUtils.read_table_columns_from_webpage_list, list_w)
for result in result_list:
stock_data_container_list.extend(result)
with open(stock_data_container_file, "wb") as f:
pickle.dump(stock_data_container_list, f)
return stock_data_container_list
def __init__(self):
self.pool = CommonUtils.get_threading_pool()