def run(self, from_date="01/01/1926", to_date=None): universe = self.asset_config for asset in universe: isin = asset['isin'] print("PROCESSING {0}".format(isin) + ":") search_results = investpy.search_etfs(by='isin', value=isin) if search_results.shape[0] > 0: try: etf_name = search_results.iloc[0]['name'] country = asset['country'] asset['name'] = etf_name asset['currency'] = search_results.iloc[0]['currency'] asset['asset_class'] = search_results.iloc[0][ 'asset_class'] asset['stock_exchange'] = search_results.iloc[0][ 'stock_exchange'] print(" Downloading data...") history = self.__get_historical_data( etf_name, country, from_date, to_date) # Only append the latest update latest_date = self.__get_latest_Update_Date(isin) history = self.__filter__(history, latest_date) print(" Writing the latest data into database...") self.__write_data(isin, history) except Exception as error: print(" ERROR: {0}".format(error)) pass print("COMPLETE") print("----------------------")
def get_data_by_isin(isin: str, dates: Tuple[datetime.date], is_etf: bool) -> Tuple[Optional[np.ndarray], str]: """Retrieves stock/ETF prices in EUR by ISIN for the given dates. Cached to make sure this is only queried once for a given currency & date-range.""" from_date = dates[0].strftime("%d/%m/%Y") to_date = (dates[-1] + datetime.timedelta(days=7)).strftime("%d/%m/%Y") # Retrieves stock/etf information based on the ISIN try: if is_etf: data = investpy.search_etfs(by="isin", value=isin) else: data = investpy.search_stocks(by="isin", value=isin) except RuntimeError: print( f"[DGPC] Warning, could not retrieve {'ETF' if is_etf else 'stock'} data for ISIN {isin}." ) return None, "" # When a stock/ETF is listed in multiple countries, take one of the preferred countries if found for country in PREFERRED_COUNTRIES: local_data = data[data["country"] == country] if local_data.shape[0] > 0: break else: # Taking the first country from the results if none of the preferred countries is found country = data["country"][0] local_data = data # Retrieves the actual historical prices for the stock/etf currency = list(local_data["currency"])[0] symbol = list(local_data["symbol"])[0] if is_etf: name = list(local_data["name"])[0] history = investpy.get_etf_historical_data(name, country=country, from_date=from_date, to_date=to_date) else: history = investpy.get_stock_historical_data(symbol, country=country, from_date=from_date, to_date=to_date) history = history.reset_index() values = densify_history(history, dates) # Convert the results to euro if currency != "EUR": currency_modifier = to_euro_modifier(currency, tuple(dates)) values *= currency_modifier return values, symbol
def import_etfs(successfulPulls): # imports ETFs in the US search_results = investpy.search_etfs(by='country', value='united states') list_of_etf_names = search_results["name"] firstIndex = datetime.datetime.strptime(_configKeys.STARTPULL, '%d/%m/%Y') lastIndex = datetime.datetime.strptime(_configKeys.ENDPULL, '%d/%m/%Y') for name in list_of_etf_names[:2500]: try: # Have an if statement in place in case if we don't want to pull every etf because there are a lot of stocks # Program takes a long time to run if we have to webscrape every etf each time we run etfData = [] etfData = investpy.get_etf_historical_data( etf=name, country='united states', from_date=_configKeys.STARTPULL, to_date=_configKeys.ENDPULL) newIndex = [] for index in etfData.index: newIndex.append( datetime.datetime.strptime( datetime.datetime.strftime((index + timedelta(days=1)), '%Y-%m-%d'), '%Y-%m-%d')) etfData['Date'] = newIndex etfData.set_index('Date', inplace=True) # If there's something that's been loaded into stockData, then the length is no longer 0 # if the differences is under 2~3 days, then it is ok to take this data since there is still enough data in the week to be usable # this timedelta fixes the problem of trying to pull during a long weekend name = str(name) + "Etf" if etfData.empty == False and etfData.index[ 0] - firstIndex <= timedelta( days=2) and lastIndex - etfData.index[-1] <= timedelta( days=3): successfulPulls["Symbol"].append(name.replace("/", "")) successfulPulls["Type"].append("ETF") etfData.to_csv( os.path.join(Path(_configKeys.DATA_FOLDER), name.replace("/", "") + '.csv')) except: print("Something went wrong when importing: " + name)
def test_etfs_errors(): """ This function raises errors on etf retrieval functions """ try: retrieve_etfs(test_mode=None) except: pass params = [ { 'country': ['error'] }, { 'country': 'error' }, ] for param in params: try: investpy.get_etfs(country=param['country']) except: pass try: investpy.get_etfs_list(country=param['country']) except: pass params = [ { 'country': ['error'], 'columns': None, 'as_json': False }, { 'country': 'spain', 'columns': None, 'as_json': 'error' }, { 'country': 'spain', 'columns': 0, 'as_json': True }, { 'country': 'spain', 'columns': ['error'], 'as_json': False }, ] for param in params: try: investpy.get_etfs_dict(country=param['country'], columns=param['columns'], as_json=param['as_json']) except: pass params = [ { 'etf': None, 'country': 'spain', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': ['error'], 'country': 'spain', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': None, 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'error', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'netherlands', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': ['error'], 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'as_json': 'error', 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'as_json': True, 'order': 'error', 'debug': True }, { 'etf': 'error', 'country': 'spain', 'as_json': True, 'order': 'ascending', 'debug': True }, { 'etf': ['error'], 'country': 'spain', 'as_json': True, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'as_json': True, 'order': 'ascending', 'debug': 'error' }, ] for param in params: try: investpy.get_etf_recent_data(etf=param['etf'], country=param['country'], as_json=param['as_json'], order=param['order'], debug=param['debug']) except: pass params = [ { 'etf': None, 'country': 'spain', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'error', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'netherlands', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': None, 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': ['error'], 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': 'error', 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'error', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': 'error', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/2019', 'to_date': 'error', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'error', 'country': 'spain', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': ['error'], 'country': 'spain', 'from_date': '01/01/2018', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/1998', 'to_date': '01/01/2019', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/2019', 'to_date': '01/01/1998', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/1900', 'to_date': '01/01/1950', 'as_json': False, 'order': 'ascending', 'debug': True }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'from_date': '01/01/2019', 'to_date': '01/03/2019', 'as_json': True, 'order': 'ascending', 'debug': 'error' }, ] for param in params: try: investpy.get_etf_historical_data(etf=param['etf'], country=param['country'], from_date=param['from_date'], to_date=param['to_date'], as_json=param['as_json'], order=param['order'], debug=param['debug']) except: pass params = [ { 'country': 'error', 'as_json': False, }, { 'country': None, 'as_json': False, }, { 'country': ['error'], 'as_json': False, }, { 'country': 'spain', 'as_json': None, }, { 'country': 'spain', 'as_json': ['error'], }, ] for param in params: try: investpy.get_etfs_overview(country=param['country']) except: pass params = [ { 'by': None, 'value': 'bbva', }, { 'by': ['error'], 'value': 'bbva', }, { 'by': 'error', 'value': 'bbva', }, { 'by': 'name', 'value': None, }, { 'by': 'name', 'value': ['error'], }, { 'by': 'isin', 'value': 'error', }, ] for param in params: try: investpy.search_etfs(by=param['by'], value=param['value']) except: pass
def test_investpy_etfs(): """ This function checks that etf data retrieval functions listed in investpy work properly. """ params = [ { 'country': 'spain', }, { 'country': None, }, ] for param in params: investpy.get_etfs(country=param['country']) investpy.get_etfs_list(country=param['country']) params = [ { 'country': None, 'columns': ['name'], 'as_json': True }, { 'country': None, 'columns': ['name'], 'as_json': False }, { 'country': 'spain', 'columns': ['name'], 'as_json': True }, { 'country': 'spain', 'columns': ['name'], 'as_json': False }, { 'country': 'spain', 'columns': None, 'as_json': False }, ] for param in params: investpy.get_etfs_dict(country=param['country'], columns=param['columns'], as_json=param['as_json']) investpy.get_etf_countries() params = [ { 'as_json': True, 'order': 'ascending', }, { 'as_json': False, 'order': 'ascending', }, { 'as_json': True, 'order': 'descending', }, { 'as_json': False, 'order': 'descending', }, ] for param in params: investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50', country='spain', as_json=param['as_json'], order=param['order'], interval='Daily') investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50', country='spain', from_date='01/01/2010', to_date='01/01/2019', as_json=param['as_json'], order=param['order'], interval='Daily') params = [ { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'as_json': False }, { 'etf': 'bbva accion dj eurostoxx 50', 'country': 'spain', 'as_json': True } ] for param in params: investpy.get_etf_information(etf=param['etf'], country=param['country'], as_json=param['as_json']) params = [ { 'country': 'united states', 'as_json': True, 'n_results': 2 }, { 'country': 'united kingdom', 'as_json': False, 'n_results': 2 }, ] for param in params: investpy.get_etfs_overview(country=param['country'], as_json=param['as_json'], n_results=param['n_results']) investpy.search_etfs(by='name', value='bbva')
def test_investpy_etfs(): """ This function checks that etf data retrieval functions listed in investpy work properly. """ params = [ { 'country': 'spain', }, { 'country': None, }, ] for param in params: investpy.get_etfs(country=param['country']) investpy.get_etfs_list(country=param['country']) params = [ { 'country': None, 'columns': ['id', 'name'], 'as_json': True }, { 'country': None, 'columns': ['id', 'name'], 'as_json': False }, { 'country': 'spain', 'columns': ['id', 'name'], 'as_json': True }, { 'country': 'spain', 'columns': ['id', 'name'], 'as_json': False }, { 'country': 'spain', 'columns': None, 'as_json': False }, ] for param in params: investpy.get_etfs_dict(country=param['country'], columns=param['columns'], as_json=param['as_json']) investpy.get_etf_countries() params = [ { 'as_json': True, 'order': 'ascending', 'debug': False }, { 'as_json': False, 'order': 'ascending', 'debug': True }, { 'as_json': True, 'order': 'descending', 'debug': False }, { 'as_json': False, 'order': 'descending', 'debug': False }, ] for param in params: investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50', country='spain', as_json=param['as_json'], order=param['order'], debug=param['debug']) investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50', country='spain', from_date='01/01/2010', to_date='01/01/2019', as_json=param['as_json'], order=param['order'], debug=param['debug']) params = [ { 'country': 'france', 'as_json': True, }, { 'country': 'france', 'as_json': False, }, ] for param in params: investpy.get_etfs_overview(country=param['country'], as_json=param['as_json']) investpy.search_etfs(by='name', value='bbva')
to_date="22/02/2020"): history = investpy.etfs.get_etf_historical_data(etf, country, from_date, to_date, as_json=False, order='ascending', interval='Daily') return history universe = assets() for asset in universe: isin = asset['isin'] print("PROCESSING {0}".format(isin) + ":") search_results = investpy.search_etfs(by='isin', value=isin) if search_results.shape[0] > 0: try: etf_name = search_results.iloc[0]['name'] country = asset['country'] asset['name'] = etf_name asset['currency'] = search_results.iloc[0]['currency'] asset['asset_class'] = search_results.iloc[0]['asset_class'] asset['stock_exchange'] = search_results.iloc[0]['stock_exchange'] print(" Downloading data...") history = get_historical_data(etf_name, country) print(" Writing data into database...") write_data(isin, history) except Exception as error: print(error)