def download_etf_information(etfs: pd.DataFrame) -> Dict:
'''Get latest etf information: latest price range, market cap etc.'''
info = {}
for i, row in etfs.iterrows():
file_name = f"etf_{row.symbol_ft}_{row['isin']}"
sleep(1)
try:
info[file_name] = investpy.get_etf_information(row['name'],
row.country,
as_json=True)
log.debug(f"Download complete (ETF information): {file_name}")
except:
log.warning(f"Download FAILED (ETF information): {file_name}")
sleep(30)
warnings.filterwarnings('default')
log.info(f"{len(info)} downloaded")
return info
def main():
parser = argparse.ArgumentParser(
description='scrap investing.com daily close')
parser.add_argument(
'-output_prefix',
type=str,
default='../stock_data/raw_daily_investing_etf/investing_etf_',
help='prefix of the output file')
parser.add_argument('-date', type=str, help='Specify the date')
args = parser.parse_args()
if args.date is None:
scrap_date = datetime.date.today()
args.date = str(scrap_date)
# get ETF overview
page = scrap_utils.get_url(
'https://www.investing.com/etfs/usa-etfs?&issuer_filter=0')
soup = bs4.BeautifulSoup(page, 'lxml')
df_overview = pd.read_html(str(soup.find('table')), header=0,
index_col=2)[0]
df_overview.drop(columns=['Unnamed: 0', 'Unnamed: 7'], inplace=True)
last_trade_time = df_overview.iloc[0]['Time']
df_overview_traded = df_overview[df_overview['Time'] == last_trade_time]
# get ETF list
etf_info_list = []
get_etfs_list = investpy.etfs.get_etfs('united states')
print('number of traded tickers:', len(df_overview_traded.index), '/',
len(get_etfs_list.index))
count = 0
for index, row in get_etfs_list.iterrows():
print('downloading...', row['symbol'], '-', count)
try:
etf_info = investpy.get_etf_information(row['name'],
'united states')
etf_info['Symbol'] = row['symbol']
etf_info_list.append(etf_info)
time.sleep(scrap_delay)
except:
print('failed')
count += 1
df_etf_info = pd.concat(etf_info_list)
df_etf_info.set_index('Symbol', inplace=True)
df = df_etf_info.join(df_overview_traded, how='left')
df['Date'] = args.date
filename = args.output_prefix + args.date + '.csv'
df.to_csv(filename)
def test_investpy_etfs():
"""
This function checks that etf data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_etfs(country=param['country'])
investpy.get_etfs_list(country=param['country'])
params = [
{
'country': None,
'columns': ['name'],
'as_json': True
},
{
'country': None,
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': ['name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_etf_countries()
params = [
{
'as_json': True,
'order': 'ascending',
},
{
'as_json': False,
'order': 'ascending',
},
{
'as_json': True,
'order': 'descending',
},
{
'as_json': False,
'order': 'descending',
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
params = [
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': False
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True
}
]
for param in params:
investpy.get_etf_information(etf=param['etf'], country=param['country'], as_json=param['as_json'])
params = [
{
'country': 'united states',
'as_json': True,
'n_results': 2
},
{
'country': 'united kingdom',
'as_json': False,
'n_results': 2
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'], as_json=param['as_json'], n_results=param['n_results'])
investpy.search_etfs(by='name', value='bbva')
req_fund_info = investpy.get_fund_information(fund=fund,
country=fund_country)
if index and index_country != "":
req_index_info = investpy.get_index_information(index=index,
country=index_country)
if stock and stock_country != "":
req_stock_info = investpy.get_stock_information(stock=stock,
country=stock_country)
if bond and bond_country != "":
req_bond_info = investpy.get_bond_information(bond=bond)
if etf and etf_country != "":
req_etf_info = investpy.get_etf_information(etf=etf, country=etf_country)
def GetFundInformation():
wb = xw.Book.caller()
wb.sheets[fund_sheet].range('N1').options(
transpose=True).value = req_fund_info
def GetIndexInformation():
wb = xw.Book.caller()
wb.sheets[index_sheet].range('N1').options(
transpose=True).value = req_index_info
def GetStockInformation():