def start_yahoo_symbol(self, min_chose_size, random_chose_size): stock_data = PyTickerSymbols() result = {} with multiprocessing.Pool(processes=self.MAX_PROCESSES * 4) as pool: for index in stock_data.get_all_indices(): args = list( sorted( map( lambda stock: Stock.from_pyticker(stock), filter( lambda x: x['name'] not in block_list_symbol_scanner, stock_data.get_stocks_by_index(index), ), ), key=lambda stock: len(stock.symbols), )) # reduce workload because of yahoo finance ban if min_chose_size + random_chose_size < len(args): args = args[:min_chose_size] + random.sample( args, random_chose_size) toolz.unique(args, key=lambda x: x.name) stocks_with_yahoo_symbols = pool.map(self.worker_yahoo, args) result[index] = stocks_with_yahoo_symbols return result
def test_index_to_yahoo(self): stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) self.assertEqual('^GDAXI', stock_data.index_to_yahoo_symbol('DAX')) self.assertEqual('^SDAXI', stock_data.index_to_yahoo_symbol('SDAX')) self.assertEqual('^MDAXI', stock_data.index_to_yahoo_symbol('MDAX')) swi = stock_data.index_to_yahoo_symbol('Switzerland 20') self.assertEqual('^SSMI', swi)
def test_encoding(self): """ Test country getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) dax = list(stock_data.get_stocks_by_index('DAX')) self.assertEqual(dax[10]['name'], 'Deutsche Börse AG')
def test_all_stocks(self): """ Test stocks getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) stocks = stock_data.get_all_stocks() self.assertTrue(len(stocks) > 900)
def __init__(self, tickers, label): ''' Constructor ''' "Initialise from a sequence" self.tickers = tickers self.label = label self.myData = [] self.stock_data = PyTickerSymbols()
def test_unique_ticker_symbols(self): stock_data = PyTickerSymbols() ctx = Counter([ sym['yahoo'] for stock in stock_data.get_all_stocks() for sym in stock['symbols'] ]) msg = 'The following symbols appear several times:\n' msg += '\n'.join( map(lambda y: y[0], filter(lambda x: x[1] > 1, ctx.items()))) self.assertFalse(any(map(lambda x: x > 1, ctx.values())), msg)
def test_indices_count(self): """ Test if all urls pressent for each index in pytickersymbols :return: """ stock_data = PyTickerSymbols() indices = stock_data.get_all_indices() is_url_list_complete = all( map(lambda x: x in Indices.symbol_source_dict, indices)) self.assertTrue(is_url_list_complete)
def home(request): ### Récuperer les tickers du CAC40 stock_data = PyTickerSymbols() try: fr_stocks = stock_data.get_stocks_by_index('CAC 40') entreprisesCAC = fr_stocks except ConnectionError as error: return error finally: return render(request, 'home.html', {'entreprisesCAC': entreprisesCAC})
def test_stock_name_by_google_symbol(self): """ Tests stock getter by industry :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) ads = stock_data.get_stock_name_by_google_symbol('FRA:ADS') self.assertIsNotNone(ads) self.assertEqual('adidas AG', ads) unknown = stock_data.get_stock_by_google_symbol('ADSdadsas.F') self.assertIsNone(unknown)
def test_tickers_valid(self): """ Test if each ticker symbol works with yfiance """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) y_tickers = stock_data.get_yahoo_ticker_symbols_by_index("DAX") for tickers in y_tickers: for ticker in tickers: y_ticker = yf.Ticker(ticker) data = y_ticker.history(period='4d') self.assertIsNotNone(data) self.assertIn("Close", data)
def symbolscanner_app(): """ Main entry point for symbolscanner application :return: """ parser = argparse.ArgumentParser(description='SymbolScanner CLI') parser.add_argument('--cache', action='store_true', default=False) parser.add_argument('--symbols', action='store_true', default=False) parser.add_argument( '-i', '--input', dest='input', action='store', help='Path to the input file.', type=lambda x: is_valid_file(parser, x), ) parser.add_argument( '-o', '--output', dest='output', action='store', help='Path to the output file.', ) args = parser.parse_args() stock_data = PyTickerSymbols() scanner = SymbolScanner(args.cache) if not args.symbols: scanner.start() else: scanner.data = scanner.start_yahoo_symbol(5, 15) indices = stock_data.get_all_indices() stocks_yaml = None with open(args.input, 'r', encoding='utf8') as in_file: stocks_yaml = yaml.safe_load(in_file) if not args.symbols: for index in indices: missing = get_missing_objects(index, scanner, stock_data) wrong = get_wrong_objects(index, scanner, stock_data) stocks_yaml = fix_wrong(wrong, index, stocks_yaml) stocks_yaml = fix_missing(missing, index, stocks_yaml) stocks_yaml = fix_symbols(stocks_yaml) stocks_yaml = update_stocks(stocks_yaml, scanner) with open(args.output, 'w') as out_file: yaml.safe_dump( stocks_yaml, out_file, sort_keys=False, encoding='utf-8', allow_unicode=True, )
def test_load_json(self): with pytest.raises(NotImplementedError): PyTickerSymbols(stocks_path='test.dat') with tempfile.NamedTemporaryFile(suffix='.json') as temp: temp.write(_test_json) temp.flush() stocks = PyTickerSymbols(stocks_path=temp.name) indices = stocks.get_all_indices() self.assertEqual(len(indices), 1) self.assertIn('test', indices) stocks = list(stocks.get_stocks_by_index('test')) self.assertEqual(len(stocks), 1) self.assertIn('name', stocks[0]) self.assertEqual('adidas AG', stocks[0].get('name', None))
def test_encoding(self): """ Test country getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) stocks = list(stock_data.get_all_stocks()) names_with_unicode_count = sum([ any(word in item['name'] for word in ['ö', 'ä', 'ü', 'é', 'è', 'ë']) for item in stocks ]) self.assertTrue(names_with_unicode_count >= 20, names_with_unicode_count)
def test_load_yaml(self): for sufix_test in ['.yaml', '.yml']: with tempfile.NamedTemporaryFile(suffix=sufix_test) as temp: temp.write( str.encode( yaml.dump(json.loads(_test_json.decode('utf-8'))))) temp.flush() stocks = PyTickerSymbols(stocks_path=temp.name) indices = stocks.get_all_indices() self.assertEqual(len(indices), 1) self.assertIn('test', indices) stocks = list(stocks.get_stocks_by_index('test')) self.assertEqual(len(stocks), 1) self.assertIn('name', stocks[0]) self.assertEqual('adidas AG', stocks[0].get('name', None))
def get_stocks(stock_index): stock_tickers = [] stock_data = PyTickerSymbols() list_stocks = stock_data.get_stocks_by_index(stock_index) #st.write(list(german_stocks)) for i in list_stocks: tickersymbol = i['symbols'][0]['yahoo'] stock_tickers.append(tickersymbol) df = web.get_quote_yahoo(stock_tickers) df.set_index('shortName', inplace=True, drop=True) st.write(df[[ 'marketCap', 'bookValue', 'price', 'priceToBook', 'fiftyDayAverage', 'fiftyTwoWeekRange', 'bidSize', 'askSize' ]]) st.write(stock_tickers)
def test_country(self): """ Test country getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) countries = list(stock_data.get_all_countries()) self.assertIsNotNone(countries) self.assertIn("Germany", countries) self.assertIn("Netherlands", countries) self.assertIn("Sweden", countries) # duplicates are not allowed for country in countries: lenl = len([tmp for tmp in countries if tmp == country]) self.assertEqual(lenl, 1)
def test_index(self): """ Test index getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) indices = stock_data.get_all_indices() self.assertIsNotNone(indices) self.assertIn("DAX", indices) self.assertIn("SDAX", indices) self.assertIn("MDAX", indices) # duplicates are not allowed for index in indices: lenl = len([tmp for tmp in indices if tmp == index]) self.assertEqual(lenl, 1)
def test_industry(self): """ Test industry getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) industries = list(stock_data.get_all_industries()) self.assertIsNotNone(industries) self.assertIn("Computer Hardware", industries) self.assertIn("Gold", industries) self.assertIn("Banking Services", industries) # duplicates are not allowed for industry in industries: lenl = len([tmp for tmp in industries if tmp == industry]) self.assertEqual(lenl, 1)
def __init__(self, arguments: dict, logger: logging.Logger): self.logger = logger self.db_args = arguments["db_args"] self.arguments = arguments self.ticker_symbols = PyTickerSymbols() # has connection try: db.bind(**self.db_args) except core.BindingError: pass else: db.generate_mapping(check_tables=False) if self.db_args.get('create_db', False): db.drop_all_tables(with_all_data=True) db.create_tables() self.__insert_initial_data()
def get_tickers(indices_names): stock_data = PyTickerSymbols() tickers = [] for i in indices_names: tickers += list(stock_data.get_stocks_by_index(i)) tickers = list(tickers) names = [] yahoo_ticker = [] industry = [] sector = [] country = [] indices = [] for i in range(len(tickers)): names.append(tickers[i]['name']) yahoo_ticker.append(tickers[i]['symbols'][0]['yahoo']) try: industry.append(tickers[i]["industries"][0]) except: industry.append('NaN') try: sector.append(tickers[i]["industries"][-1]) except: sector.append('NaN') country.append(tickers[i]["country"]) indices.append(tickers[i]["indices"]) tickers_df = pd.DataFrame() tickers_df['name'] = names tickers_df['yahoo_ticker'] = yahoo_ticker tickers_df['industry'] = industry tickers_df['sector'] = sector tickers_df['country'] = country tickers_df['indices'] = indices tickers_df.index = yahoo_ticker tickers_df = tickers_df.drop_duplicates(subset=['yahoo_ticker']) return tickers_df
def test_valid_country_name(self): stock_data = PyTickerSymbols() countries = stock_data.get_all_countries() empty_names = list(filter(lambda x: not x, countries)) empty_country_stocks = ', '.join( list( map( lambda x: x['name'], stock_data.get_stocks_by_country(''), ))) self.assertEqual( len(empty_names), 0, 'The following stocks have an empty country string: ' + empty_country_stocks, ) valid_countires = list(map( lambda x: x.name, pycountry.countries, )) wrong_country_name = list( filter( lambda x: x['country'] not in valid_countires, stock_data.get_all_stocks(), ), ) wrong_country_name_stocks = ', '.join( list( map( lambda x: x['name'] + '(' + x['country'] + ')', wrong_country_name, ))) self.assertEqual( len(wrong_country_name), 0, 'The following stocks have an empty country string:' + wrong_country_name_stocks, )
def test_dynamic_methods(self): """ Test dynamic ticker getter :return: """ stock_data = PyTickerSymbols() dax_google = stock_data.get_dax_frankfurt_google_tickers() dax_yahoo = stock_data.get_dax_frankfurt_yahoo_tickers() self.assertTrue(all(map(lambda x: x.startswith('FRA:'), dax_google))) self.assertTrue(len(dax_google) > 40) self.assertTrue(all(map(lambda x: x.endswith('.F'), dax_yahoo))) self.assertTrue(len(dax_yahoo) > 40) myvars = dir(stock_data) for method in filter( lambda x: (x.endswith('_google_tickers') or x.endswith('_yahoo_tickers') ) and '_moscow' not in x and '_cdax' not in x, myvars, ): result = getattr(stock_data, method)() self.assertTrue( len(result) > 5, f'{method} has only {len(result)} tickers')
def test_stocks_by_index(self): """ Tests stock getter :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) stocks = list(stock_data.get_stocks_by_index(None)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_index(False)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_index(True)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_index(22)) self.assertEqual(len(stocks), 0) for ind, ctx in [('DAX', 30), ('CAC 40', 40)]: stocks = list(stock_data.get_stocks_by_index(ind)) self.assertIsNotNone(stocks) self.assertEqual(len(stocks), ctx) for stock in stocks: is_in = False for index in stock["indices"]: if ind in index: is_in = True self.assertTrue(is_in) # test NASDAQ 100 stocks_nasdaq = list(stock_data.get_stocks_by_index('NASDAQ 100')) stocks_nasdaq_symbol = [ sym['yahoo'] for stock in stocks_nasdaq for sym in stock['symbols'] ] symbols_nasdaq = list( stock_data.get_yahoo_ticker_symbols_by_index('NASDAQ 100')) symbols_nasdaq = reduce(lambda x, y: x + y, symbols_nasdaq) self.assertEqual(len(stocks_nasdaq_symbol), len(symbols_nasdaq)) self.assertIn('GOOGL', symbols_nasdaq) self.assertIn('GOOG', symbols_nasdaq) y_ticker = yf.Ticker('GOOG') data = y_ticker.history(period='4d') self.assertIsNotNone(data) y_ticker = yf.Ticker('GOOGL') data = y_ticker.history(period='4d') self.assertIsNotNone(data)
def start_sync(self): stocks = PyTickerSymbols() if not self.data: return stocks_py = stocks.get_all_stocks() names_py = list(set([stock['name'] for stock in stocks_py])) names_wiki = list( set([ stock.wiki_name for stock_list in map( lambda index: self.data[index], Indices.symbol_source_dict, ) for stock in stock_list ])) names = names_py + names_wiki endings = list( map(lambda x: ' ' + x, self.get_most_common_endings(names))) occurrences = list( set( list(self.get_most_common_occurrences(names)) + most_common_endings + endings)) occurrences.sort(key=len, reverse=True) endings.sort(key=len, reverse=True) with multiprocessing.Pool(processes=self.MAX_PROCESSES) as pool: items = list( map( lambda index: (index, endings, occurrences, stocks_py), Indices.symbol_source_dict, )) sync_results = pool.map(self.worker_sync, items) result = dict(map(lambda val: list(val.items())[0], sync_results)) return result
def esg_firm_query_keywords_pipeline(index_name, path_to_settings): """ESG scores, processed firm names and firm name query strings in a dataframe. Args: index_name (string): Index name, one of PyTickerSymbols().get_all_indices() path_to_settings (string): path to settings.yaml, where all esg keywords are specified Returns: Dataframe: esg scores and related data from Yahoo!Finance incl. processed firm names and query keywords """ pytickersymbols = PyTickerSymbols() controversy_keywords = get_esg_controversy_keywords(path_to_settings) esg_df = (get_index_firm_esg( pytickersymbols=pytickersymbols, index_name=index_name).pipe(replace_firm_names, settings_path=path_to_settings).pipe( remove_missing_esg_firms).pipe( create_query_keywords, keyword_list=controversy_keywords)) return esg_df
def test_stocks_by_country(self): """ Tests stock getter by country :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) stocks = list(stock_data.get_stocks_by_country(None)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_country(False)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_country(True)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_country(22)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_country("Israel")) self.assertIsNotNone(stocks) self.assertTrue(len(stocks) >= 1) for stock in stocks: is_in_israel = False if "Israel" == stock["country"]: is_in_israel = True self.assertTrue(is_in_israel)
def test_stocks_by_industry(self): """ Tests stock getter by industry :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) stocks = list(stock_data.get_stocks_by_industry(None)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_industry(False)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_industry(True)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_industry(22)) self.assertEqual(len(stocks), 0) stocks = list(stock_data.get_stocks_by_industry("Basic Materials")) self.assertIsNotNone(stocks) for stock in stocks: is_in_basic = False for industry in stock["industries"]: if "Basic Materials" in industry: is_in_basic = True self.assertTrue(is_in_basic)
def __init__(self, skip_fix_symbols=False): self.ticker_symbols = None if not skip_fix_symbols: self.ticker_symbols = PyTickerSymbols()
def test_tickers_by_index(self): """ Tests tickers getter by index :return: """ stock_data = PyTickerSymbols() self.assertIsNotNone(stock_data) google_tickers = stock_data.get_google_ticker_symbols_by_index(None) self.assertEqual(len(google_tickers), 0) google_tickers = stock_data.get_google_ticker_symbols_by_index(False) self.assertEqual(len(google_tickers), 0) google_tickers = stock_data.get_google_ticker_symbols_by_index(True) self.assertEqual(len(google_tickers), 0) google_tickers = stock_data.get_google_ticker_symbols_by_index(22) self.assertEqual(len(google_tickers), 0) google_tickers = stock_data.get_google_ticker_symbols_by_index("DAX") self.assertIsNotNone(google_tickers) yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(None) self.assertEqual(len(yahoo_tickers), 0) yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(False) self.assertEqual(len(yahoo_tickers), 0) yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(True) self.assertEqual(len(yahoo_tickers), 0) yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index(22) self.assertEqual(len(yahoo_tickers), 0) yahoo_tickers = stock_data.get_yahoo_ticker_symbols_by_index("DAX") self.assertIsNotNone(yahoo_tickers) test_list = [google_tickers, yahoo_tickers] for test_item in test_list: self.assertEqual(len(test_item), 30) for tickers in test_item: self.assertEqual(len(tickers), 2)
def test_singleton(self): """ Test singleton pattern :return: """ self.assertTrue(id(PyTickerSymbols()) == id(PyTickerSymbols()))