def test_load_json(self):
with pytest.raises(NotImplementedError):
PyTickerSymbols(stocks_path='test.dat')
with tempfile.NamedTemporaryFile(suffix='.json') as temp:
temp.write(_test_json)
temp.flush()
stocks = PyTickerSymbols(stocks_path=temp.name)
indices = stocks.get_all_indices()
self.assertEqual(len(indices), 1)
self.assertIn('test', indices)
stocks = list(stocks.get_stocks_by_index('test'))
self.assertEqual(len(stocks), 1)
self.assertIn('name', stocks[0])
self.assertEqual('adidas AG', stocks[0].get('name', None))
def start_yahoo_symbol(self, min_chose_size, random_chose_size):
stock_data = PyTickerSymbols()
result = {}
with multiprocessing.Pool(processes=self.MAX_PROCESSES * 4) as pool:
for index in stock_data.get_all_indices():
args = list(
sorted(
map(
lambda stock: Stock.from_pyticker(stock),
filter(
lambda x: x['name'] not in
block_list_symbol_scanner,
stock_data.get_stocks_by_index(index),
),
),
key=lambda stock: len(stock.symbols),
))
# reduce workload because of yahoo finance ban
if min_chose_size + random_chose_size < len(args):
args = args[:min_chose_size] + random.sample(
args, random_chose_size)
toolz.unique(args, key=lambda x: x.name)
stocks_with_yahoo_symbols = pool.map(self.worker_yahoo, args)
result[index] = stocks_with_yahoo_symbols
return result
def test_tickers_by_index(self):
"""
Tests tickers getter by index
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
google_tickers = stock_data.get_google_ticker_symbols_by_index(None)
self.assertEqual(len(google_tickers), 0)
google_tickers = stock_data.get_google_ticker_symbols_by_index(False)
self.assertEqual(len(google_tickers), 0)
google_tickers = stock_data.get_google_ticker_symbols_by_index(True)
self.assertEqual(len(google_tickers), 0)
google_tickers = stock_data.get_google_ticker_symbols_by_index(22)
self.assertEqual(len(google_tickers), 0)
google_tickers = stock_data.get_google_ticker_symbols_by_index("DAX")
self.assertIsNotNone(google_tickers)
yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(None)
self.assertEqual(len(yahoo_tickers), 0)
yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(False)
self.assertEqual(len(yahoo_tickers), 0)
yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(True)
self.assertEqual(len(yahoo_tickers), 0)
yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(22)
self.assertEqual(len(yahoo_tickers), 0)
yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index("DAX")
self.assertIsNotNone(yahoo_tickers)
test_list = [google_tickers, yahoo_tickers]
for test_item in test_list:
self.assertEqual(len(test_item), 30)
for tickers in test_item:
self.assertEqual(len(tickers), 2)
def test_index_to_yahoo(self):
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
self.assertEqual('^GDAXI', stock_data.index_to_yahoo_symbol('DAX'))
self.assertEqual('^SDAXI', stock_data.index_to_yahoo_symbol('SDAX'))
self.assertEqual('^MDAXI', stock_data.index_to_yahoo_symbol('MDAX'))
swi = stock_data.index_to_yahoo_symbol('Switzerland 20')
self.assertEqual('^SSMI', swi)
def test_all_stocks(self):
"""
Test stocks getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
stocks = stock_data.get_all_stocks()
self.assertTrue(len(stocks) > 900)
def test_encoding(self):
"""
Test country getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
dax = list(stock_data.get_stocks_by_index('DAX'))
self.assertEqual(dax[10]['name'], 'Deutsche Börse AG')
def __init__(self, tickers, label):
'''
Constructor
'''
"Initialise from a sequence"
self.tickers = tickers
self.label = label
self.myData = []
self.stock_data = PyTickerSymbols()
def test_indices_count(self):
"""
Test if all urls pressent for each index in pytickersymbols
:return:
"""
stock_data = PyTickerSymbols()
indices = stock_data.get_all_indices()
is_url_list_complete = all(
map(lambda x: x in Indices.symbol_source_dict, indices))
self.assertTrue(is_url_list_complete)
def home(request):
### Récuperer les tickers du CAC40
stock_data = PyTickerSymbols()
try:
fr_stocks = stock_data.get_stocks_by_index('CAC 40')
entreprisesCAC = fr_stocks
except ConnectionError as error:
return error
finally:
return render(request, 'home.html', {'entreprisesCAC': entreprisesCAC})
def test_unique_ticker_symbols(self):
stock_data = PyTickerSymbols()
ctx = Counter([
sym['yahoo'] for stock in stock_data.get_all_stocks()
for sym in stock['symbols']
])
msg = 'The following symbols appear several times:\n'
msg += '\n'.join(
map(lambda y: y[0], filter(lambda x: x[1] > 1, ctx.items())))
self.assertFalse(any(map(lambda x: x > 1, ctx.values())), msg)
def test_stock_name_by_google_symbol(self):
"""
Tests stock getter by industry
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
ads = stock_data.get_stock_name_by_google_symbol('FRA:ADS')
self.assertIsNotNone(ads)
self.assertEqual('adidas AG', ads)
unknown = stock_data.get_stock_by_google_symbol('ADSdadsas.F')
self.assertIsNone(unknown)
def test_tickers_valid(self):
"""
Test if each ticker symbol works with yfiance
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
y_tickers = stock_data.get_yahoo_ticker_symbols_by_index("DAX")
for tickers in y_tickers:
for ticker in tickers:
y_ticker = yf.Ticker(ticker)
data = y_ticker.history(period='4d')
self.assertIsNotNone(data)
self.assertIn("Close", data)
def symbolscanner_app():
"""
Main entry point for symbolscanner application
:return:
"""
parser = argparse.ArgumentParser(description='SymbolScanner CLI')
parser.add_argument('--cache', action='store_true', default=False)
parser.add_argument('--symbols', action='store_true', default=False)
parser.add_argument(
'-i',
'--input',
dest='input',
action='store',
help='Path to the input file.',
type=lambda x: is_valid_file(parser, x),
)
parser.add_argument(
'-o',
'--output',
dest='output',
action='store',
help='Path to the output file.',
)
args = parser.parse_args()
stock_data = PyTickerSymbols()
scanner = SymbolScanner(args.cache)
if not args.symbols:
scanner.start()
else:
scanner.data = scanner.start_yahoo_symbol(5, 15)
indices = stock_data.get_all_indices()
stocks_yaml = None
with open(args.input, 'r', encoding='utf8') as in_file:
stocks_yaml = yaml.safe_load(in_file)
if not args.symbols:
for index in indices:
missing = get_missing_objects(index, scanner, stock_data)
wrong = get_wrong_objects(index, scanner, stock_data)
stocks_yaml = fix_wrong(wrong, index, stocks_yaml)
stocks_yaml = fix_missing(missing, index, stocks_yaml)
stocks_yaml = fix_symbols(stocks_yaml)
stocks_yaml = update_stocks(stocks_yaml, scanner)
with open(args.output, 'w') as out_file:
yaml.safe_dump(
stocks_yaml,
out_file,
sort_keys=False,
encoding='utf-8',
allow_unicode=True,
)
def test_encoding(self):
"""
Test country getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
stocks = list(stock_data.get_all_stocks())
names_with_unicode_count = sum([
any(word in item['name']
for word in ['ö', 'ä', 'ü', 'é', 'è', 'ë']) for item in stocks
])
self.assertTrue(names_with_unicode_count >= 20,
names_with_unicode_count)
def get_stocks(stock_index):
stock_tickers = []
stock_data = PyTickerSymbols()
list_stocks = stock_data.get_stocks_by_index(stock_index)
#st.write(list(german_stocks))
for i in list_stocks:
tickersymbol = i['symbols'][0]['yahoo']
stock_tickers.append(tickersymbol)
df = web.get_quote_yahoo(stock_tickers)
df.set_index('shortName', inplace=True, drop=True)
st.write(df[[
'marketCap', 'bookValue', 'price', 'priceToBook', 'fiftyDayAverage',
'fiftyTwoWeekRange', 'bidSize', 'askSize'
]])
st.write(stock_tickers)
def test_load_yaml(self):
for sufix_test in ['.yaml', '.yml']:
with tempfile.NamedTemporaryFile(suffix=sufix_test) as temp:
temp.write(
str.encode(
yaml.dump(json.loads(_test_json.decode('utf-8')))))
temp.flush()
stocks = PyTickerSymbols(stocks_path=temp.name)
indices = stocks.get_all_indices()
self.assertEqual(len(indices), 1)
self.assertIn('test', indices)
stocks = list(stocks.get_stocks_by_index('test'))
self.assertEqual(len(stocks), 1)
self.assertIn('name', stocks[0])
self.assertEqual('adidas AG', stocks[0].get('name', None))
def test_industry(self):
"""
Test industry getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
industries = list(stock_data.get_all_industries())
self.assertIsNotNone(industries)
self.assertIn("Computer Hardware", industries)
self.assertIn("Gold", industries)
self.assertIn("Banking Services", industries)
# duplicates are not allowed
for industry in industries:
lenl = len([tmp for tmp in industries if tmp == industry])
self.assertEqual(lenl, 1)
def test_index(self):
"""
Test index getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
indices = stock_data.get_all_indices()
self.assertIsNotNone(indices)
self.assertIn("DAX", indices)
self.assertIn("SDAX", indices)
self.assertIn("MDAX", indices)
# duplicates are not allowed
for index in indices:
lenl = len([tmp for tmp in indices if tmp == index])
self.assertEqual(lenl, 1)
def test_country(self):
"""
Test country getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
countries = list(stock_data.get_all_countries())
self.assertIsNotNone(countries)
self.assertIn("Germany", countries)
self.assertIn("Netherlands", countries)
self.assertIn("Sweden", countries)
# duplicates are not allowed
for country in countries:
lenl = len([tmp for tmp in countries if tmp == country])
self.assertEqual(lenl, 1)
def __init__(self, arguments: dict, logger: logging.Logger):
self.logger = logger
self.db_args = arguments["db_args"]
self.arguments = arguments
self.ticker_symbols = PyTickerSymbols()
# has connection
try:
db.bind(**self.db_args)
except core.BindingError:
pass
else:
db.generate_mapping(check_tables=False)
if self.db_args.get('create_db', False):
db.drop_all_tables(with_all_data=True)
db.create_tables()
self.__insert_initial_data()
def test_stocks_by_index(self):
"""
Tests stock getter
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
stocks = list(stock_data.get_stocks_by_index(None))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_index(False))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_index(True))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_index(22))
self.assertEqual(len(stocks), 0)
for ind, ctx in [('DAX', 30), ('CAC 40', 40)]:
stocks = list(stock_data.get_stocks_by_index(ind))
self.assertIsNotNone(stocks)
self.assertEqual(len(stocks), ctx)
for stock in stocks:
is_in = False
for index in stock["indices"]:
if ind in index:
is_in = True
self.assertTrue(is_in)
# test NASDAQ 100
stocks_nasdaq = list(stock_data.get_stocks_by_index('NASDAQ 100'))
stocks_nasdaq_symbol = [
sym['yahoo'] for stock in stocks_nasdaq for sym in stock['symbols']
]
symbols_nasdaq = list(
stock_data.get_yahoo_ticker_symbols_by_index('NASDAQ 100'))
symbols_nasdaq = reduce(lambda x, y: x + y, symbols_nasdaq)
self.assertEqual(len(stocks_nasdaq_symbol), len(symbols_nasdaq))
self.assertIn('GOOGL', symbols_nasdaq)
self.assertIn('GOOG', symbols_nasdaq)
y_ticker = yf.Ticker('GOOG')
data = y_ticker.history(period='4d')
self.assertIsNotNone(data)
y_ticker = yf.Ticker('GOOGL')
data = y_ticker.history(period='4d')
self.assertIsNotNone(data)
def get_tickers(indices_names):
stock_data = PyTickerSymbols()
tickers = []
for i in indices_names:
tickers += list(stock_data.get_stocks_by_index(i))
tickers = list(tickers)
names = []
yahoo_ticker = []
industry = []
sector = []
country = []
indices = []
for i in range(len(tickers)):
names.append(tickers[i]['name'])
yahoo_ticker.append(tickers[i]['symbols'][0]['yahoo'])
try:
industry.append(tickers[i]["industries"][0])
except:
industry.append('NaN')
try:
sector.append(tickers[i]["industries"][-1])
except:
sector.append('NaN')
country.append(tickers[i]["country"])
indices.append(tickers[i]["indices"])
tickers_df = pd.DataFrame()
tickers_df['name'] = names
tickers_df['yahoo_ticker'] = yahoo_ticker
tickers_df['industry'] = industry
tickers_df['sector'] = sector
tickers_df['country'] = country
tickers_df['indices'] = indices
tickers_df.index = yahoo_ticker
tickers_df = tickers_df.drop_duplicates(subset=['yahoo_ticker'])
return tickers_df
def esg_firm_query_keywords_pipeline(index_name, path_to_settings):
"""ESG scores, processed firm names and firm name query strings in a dataframe.
Args:
index_name (string): Index name, one of PyTickerSymbols().get_all_indices()
path_to_settings (string): path to settings.yaml, where all esg keywords are specified
Returns:
Dataframe: esg scores and related data from Yahoo!Finance incl. processed firm names and query keywords
"""
pytickersymbols = PyTickerSymbols()
controversy_keywords = get_esg_controversy_keywords(path_to_settings)
esg_df = (get_index_firm_esg(
pytickersymbols=pytickersymbols,
index_name=index_name).pipe(replace_firm_names,
settings_path=path_to_settings).pipe(
remove_missing_esg_firms).pipe(
create_query_keywords,
keyword_list=controversy_keywords))
return esg_df
def test_valid_country_name(self):
stock_data = PyTickerSymbols()
countries = stock_data.get_all_countries()
empty_names = list(filter(lambda x: not x, countries))
empty_country_stocks = ', '.join(
list(
map(
lambda x: x['name'],
stock_data.get_stocks_by_country(''),
)))
self.assertEqual(
len(empty_names),
0,
'The following stocks have an empty country string: ' +
empty_country_stocks,
)
valid_countires = list(map(
lambda x: x.name,
pycountry.countries,
))
wrong_country_name = list(
filter(
lambda x: x['country'] not in valid_countires,
stock_data.get_all_stocks(),
), )
wrong_country_name_stocks = ', '.join(
list(
map(
lambda x: x['name'] + '(' + x['country'] + ')',
wrong_country_name,
)))
self.assertEqual(
len(wrong_country_name),
0,
'The following stocks have an empty country string:' +
wrong_country_name_stocks,
)
def test_dynamic_methods(self):
"""
Test dynamic ticker getter
:return:
"""
stock_data = PyTickerSymbols()
dax_google = stock_data.get_dax_frankfurt_google_tickers()
dax_yahoo = stock_data.get_dax_frankfurt_yahoo_tickers()
self.assertTrue(all(map(lambda x: x.startswith('FRA:'), dax_google)))
self.assertTrue(len(dax_google) > 40)
self.assertTrue(all(map(lambda x: x.endswith('.F'), dax_yahoo)))
self.assertTrue(len(dax_yahoo) > 40)
myvars = dir(stock_data)
for method in filter(
lambda x:
(x.endswith('_google_tickers') or x.endswith('_yahoo_tickers')
) and '_moscow' not in x and '_cdax' not in x,
myvars,
):
result = getattr(stock_data, method)()
self.assertTrue(
len(result) > 5, f'{method} has only {len(result)} tickers')
def test_stocks_by_country(self):
"""
Tests stock getter by country
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
stocks = list(stock_data.get_stocks_by_country(None))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_country(False))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_country(True))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_country(22))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_country("Israel"))
self.assertIsNotNone(stocks)
self.assertTrue(len(stocks) >= 1)
for stock in stocks:
is_in_israel = False
if "Israel" == stock["country"]:
is_in_israel = True
self.assertTrue(is_in_israel)
def test_stocks_by_industry(self):
"""
Tests stock getter by industry
:return:
"""
stock_data = PyTickerSymbols()
self.assertIsNotNone(stock_data)
stocks = list(stock_data.get_stocks_by_industry(None))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_industry(False))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_industry(True))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_industry(22))
self.assertEqual(len(stocks), 0)
stocks = list(stock_data.get_stocks_by_industry("Basic Materials"))
self.assertIsNotNone(stocks)
for stock in stocks:
is_in_basic = False
for industry in stock["industries"]:
if "Basic Materials" in industry:
is_in_basic = True
self.assertTrue(is_in_basic)
def start_sync(self):
stocks = PyTickerSymbols()
if not self.data:
return
stocks_py = stocks.get_all_stocks()
names_py = list(set([stock['name'] for stock in stocks_py]))
names_wiki = list(
set([
stock.wiki_name for stock_list in map(
lambda index: self.data[index],
Indices.symbol_source_dict,
) for stock in stock_list
]))
names = names_py + names_wiki
endings = list(
map(lambda x: ' ' + x, self.get_most_common_endings(names)))
occurrences = list(
set(
list(self.get_most_common_occurrences(names)) +
most_common_endings + endings))
occurrences.sort(key=len, reverse=True)
endings.sort(key=len, reverse=True)
with multiprocessing.Pool(processes=self.MAX_PROCESSES) as pool:
items = list(
map(
lambda index: (index, endings, occurrences, stocks_py),
Indices.symbol_source_dict,
))
sync_results = pool.map(self.worker_sync, items)
result = dict(map(lambda val: list(val.items())[0], sync_results))
return result
def __init__(self, skip_fix_symbols=False):
self.ticker_symbols = None
if not skip_fix_symbols:
self.ticker_symbols = PyTickerSymbols()
def test_singleton(self):
"""
Test singleton pattern
:return:
"""
self.assertTrue(id(PyTickerSymbols()) == id(PyTickerSymbols()))