def run(self, from_date="01/01/1926", to_date=None):
universe = self.asset_config
for asset in universe:
isin = asset['isin']
print("PROCESSING {0}".format(isin) + ":")
search_results = investpy.search_etfs(by='isin', value=isin)
if search_results.shape[0] > 0:
try:
etf_name = search_results.iloc[0]['name']
country = asset['country']
asset['name'] = etf_name
asset['currency'] = search_results.iloc[0]['currency']
asset['asset_class'] = search_results.iloc[0][
'asset_class']
asset['stock_exchange'] = search_results.iloc[0][
'stock_exchange']
print(" Downloading data...")
history = self.__get_historical_data(
etf_name, country, from_date, to_date)
# Only append the latest update
latest_date = self.__get_latest_Update_Date(isin)
history = self.__filter__(history, latest_date)
print(" Writing the latest data into database...")
self.__write_data(isin, history)
except Exception as error:
print(" ERROR: {0}".format(error))
pass
print("COMPLETE")
print("----------------------")
def get_data_by_isin(isin: str, dates: Tuple[datetime.date],
is_etf: bool) -> Tuple[Optional[np.ndarray], str]:
"""Retrieves stock/ETF prices in EUR by ISIN for the given dates. Cached to make sure this is only queried once for
a given currency & date-range."""
from_date = dates[0].strftime("%d/%m/%Y")
to_date = (dates[-1] + datetime.timedelta(days=7)).strftime("%d/%m/%Y")
# Retrieves stock/etf information based on the ISIN
try:
if is_etf:
data = investpy.search_etfs(by="isin", value=isin)
else:
data = investpy.search_stocks(by="isin", value=isin)
except RuntimeError:
print(
f"[DGPC] Warning, could not retrieve {'ETF' if is_etf else 'stock'} data for ISIN {isin}."
)
return None, ""
# When a stock/ETF is listed in multiple countries, take one of the preferred countries if found
for country in PREFERRED_COUNTRIES:
local_data = data[data["country"] == country]
if local_data.shape[0] > 0:
break
else:
# Taking the first country from the results if none of the preferred countries is found
country = data["country"][0]
local_data = data
# Retrieves the actual historical prices for the stock/etf
currency = list(local_data["currency"])[0]
symbol = list(local_data["symbol"])[0]
if is_etf:
name = list(local_data["name"])[0]
history = investpy.get_etf_historical_data(name,
country=country,
from_date=from_date,
to_date=to_date)
else:
history = investpy.get_stock_historical_data(symbol,
country=country,
from_date=from_date,
to_date=to_date)
history = history.reset_index()
values = densify_history(history, dates)
# Convert the results to euro
if currency != "EUR":
currency_modifier = to_euro_modifier(currency, tuple(dates))
values *= currency_modifier
return values, symbol
def import_etfs(successfulPulls):
# imports ETFs in the US
search_results = investpy.search_etfs(by='country', value='united states')
list_of_etf_names = search_results["name"]
firstIndex = datetime.datetime.strptime(_configKeys.STARTPULL, '%d/%m/%Y')
lastIndex = datetime.datetime.strptime(_configKeys.ENDPULL, '%d/%m/%Y')
for name in list_of_etf_names[:2500]:
try:
# Have an if statement in place in case if we don't want to pull every etf because there are a lot of stocks
# Program takes a long time to run if we have to webscrape every etf each time we run
etfData = []
etfData = investpy.get_etf_historical_data(
etf=name,
country='united states',
from_date=_configKeys.STARTPULL,
to_date=_configKeys.ENDPULL)
newIndex = []
for index in etfData.index:
newIndex.append(
datetime.datetime.strptime(
datetime.datetime.strftime((index + timedelta(days=1)),
'%Y-%m-%d'), '%Y-%m-%d'))
etfData['Date'] = newIndex
etfData.set_index('Date', inplace=True)
# If there's something that's been loaded into stockData, then the length is no longer 0
# if the differences is under 2~3 days, then it is ok to take this data since there is still enough data in the week to be usable
# this timedelta fixes the problem of trying to pull during a long weekend
name = str(name) + "Etf"
if etfData.empty == False and etfData.index[
0] - firstIndex <= timedelta(
days=2) and lastIndex - etfData.index[-1] <= timedelta(
days=3):
successfulPulls["Symbol"].append(name.replace("/", ""))
successfulPulls["Type"].append("ETF")
etfData.to_csv(
os.path.join(Path(_configKeys.DATA_FOLDER),
name.replace("/", "") + '.csv'))
except:
print("Something went wrong when importing: " + name)
def test_etfs_errors():
"""
This function raises errors on etf retrieval functions
"""
try:
retrieve_etfs(test_mode=None)
except:
pass
params = [
{
'country': ['error']
},
{
'country': 'error'
},
]
for param in params:
try:
investpy.get_etfs(country=param['country'])
except:
pass
try:
investpy.get_etfs_list(country=param['country'])
except:
pass
params = [
{
'country': ['error'],
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': 'error'
},
{
'country': 'spain',
'columns': 0,
'as_json': True
},
{
'country': 'spain',
'columns': ['error'],
'as_json': False
},
]
for param in params:
try:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
except:
pass
params = [
{
'etf': None,
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': None,
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'netherlands',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': ['error'],
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True,
'order': 'error',
'debug': True
},
{
'etf': 'error',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_etf_recent_data(etf=param['etf'],
country=param['country'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'etf': None,
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'error',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'netherlands',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': None,
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': ['error'],
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'error',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': 'error',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'error',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/1998',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/01/1998',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/1900',
'to_date': '01/01/1950',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/03/2019',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_etf_historical_data(etf=param['etf'],
country=param['country'],
from_date=param['from_date'],
to_date=param['to_date'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'country': 'error',
'as_json': False,
},
{
'country': None,
'as_json': False,
},
{
'country': ['error'],
'as_json': False,
},
{
'country': 'spain',
'as_json': None,
},
{
'country': 'spain',
'as_json': ['error'],
},
]
for param in params:
try:
investpy.get_etfs_overview(country=param['country'])
except:
pass
params = [
{
'by': None,
'value': 'bbva',
},
{
'by': ['error'],
'value': 'bbva',
},
{
'by': 'error',
'value': 'bbva',
},
{
'by': 'name',
'value': None,
},
{
'by': 'name',
'value': ['error'],
},
{
'by': 'isin',
'value': 'error',
},
]
for param in params:
try:
investpy.search_etfs(by=param['by'], value=param['value'])
except:
pass
def test_investpy_etfs():
"""
This function checks that etf data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_etfs(country=param['country'])
investpy.get_etfs_list(country=param['country'])
params = [
{
'country': None,
'columns': ['name'],
'as_json': True
},
{
'country': None,
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': ['name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_etf_countries()
params = [
{
'as_json': True,
'order': 'ascending',
},
{
'as_json': False,
'order': 'ascending',
},
{
'as_json': True,
'order': 'descending',
},
{
'as_json': False,
'order': 'descending',
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
params = [
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': False
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True
}
]
for param in params:
investpy.get_etf_information(etf=param['etf'], country=param['country'], as_json=param['as_json'])
params = [
{
'country': 'united states',
'as_json': True,
'n_results': 2
},
{
'country': 'united kingdom',
'as_json': False,
'n_results': 2
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'], as_json=param['as_json'], n_results=param['n_results'])
investpy.search_etfs(by='name', value='bbva')
def test_investpy_etfs():
"""
This function checks that etf data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_etfs(country=param['country'])
investpy.get_etfs_list(country=param['country'])
params = [
{
'country': None,
'columns': ['id', 'name'],
'as_json': True
},
{
'country': None,
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_etf_countries()
params = [
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
params = [
{
'country': 'france',
'as_json': True,
},
{
'country': 'france',
'as_json': False,
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'],
as_json=param['as_json'])
investpy.search_etfs(by='name', value='bbva')
to_date="22/02/2020"):
history = investpy.etfs.get_etf_historical_data(etf,
country,
from_date,
to_date,
as_json=False,
order='ascending',
interval='Daily')
return history
universe = assets()
for asset in universe:
isin = asset['isin']
print("PROCESSING {0}".format(isin) + ":")
search_results = investpy.search_etfs(by='isin', value=isin)
if search_results.shape[0] > 0:
try:
etf_name = search_results.iloc[0]['name']
country = asset['country']
asset['name'] = etf_name
asset['currency'] = search_results.iloc[0]['currency']
asset['asset_class'] = search_results.iloc[0]['asset_class']
asset['stock_exchange'] = search_results.iloc[0]['stock_exchange']
print(" Downloading data...")
history = get_historical_data(etf_name, country)
print(" Writing data into database...")
write_data(isin, history)
except Exception as error:
print(error)
