def main():
parser = argparse.ArgumentParser(description='scrap yahoo earning')
parser.add_argument('-output_prefix', type=str, default='data_tickers/investing_', help='output file prefix')
args = parser.parse_args()
# Index
indices = investpy.get_indices()
df_index = pd.DataFrame(indices)
df_index.to_csv(args.output_prefix + 'index_info.csv')
# Commodity
commodities = investpy.get_commodities()
commodities_dict = investpy.get_commodities_dict()
commodity_info_list = []
for commodity,country in zip(commodities['name'].to_list(),commodities['country'].to_list()):
print(commodity,',',country)
commodity_info_list.append(investpy.get_commodity_information(commodity,country))
time.sleep(scrap_delay)
df_commodity = pd.concat(commodity_info_list)
df_commodity.to_csv(args.output_prefix + 'commodity_info.csv')
# ETF
etfs_dict_us = investpy.get_etfs_dict('united states')
etfs_dict_canada = investpy.get_etfs_dict('canada')
df_etfs = pd.DataFrame(etfs_dict_us + etfs_dict_canada)
df_etfs.to_csv(args.output_prefix + 'etfs_info.csv')
# Stock
stocks_dict_us = investpy.get_stocks_dict('united states')
stocks_dict_canada = investpy.get_stocks_dict('canada')
df_stocks = pd.DataFrame(stocks_dict_us + stocks_dict_canada)
df_stocks.to_csv(args.output_prefix + 'stock_info.csv')
def main():
parser = argparse.ArgumentParser(description='scrap yahoo earning')
parser.add_argument(
'-output_dir',
type=str,
default='../stock_data_local/raw_history_investing_etf/',
help='output directory')
parser.add_argument('-skip', type=int, help='skip tickers')
args = parser.parse_args()
country = 'united states'
etfs_dict = investpy.get_etfs_dict(country)
today = datetime.datetime.now().strftime('%d/%m/%Y')
for count, etf_row in enumerate(etfs_dict):
if args.skip is not None:
if count < args.skip:
continue
ticker = etf_row['symbol']
print('downloading...', ticker, '-', count, '/', len(etfs_dict))
try:
df_etf = investpy.get_etf_historical_data(etf_row['name'], country,
'1/1/1990', today)
df_etf.to_csv(args.output_dir + ticker + '.csv')
except Exception as e:
print('FAIL -', e)
time.sleep(scrap_delay)
return 0
def getStockStaticData(isin = None, currency = None):
global __LOCAL_STATIC_CACHE
if not __LOCAL_STATIC_CACHE:
etfs = investpy.get_etfs_dict()
for x in etfs:
x['type'] = 'ETF'
__LOCAL_STATIC_CACHE += etfs
stocks = investpy.get_stocks_dict()
for x in stocks:
x['type'] = 'Equity'
__LOCAL_STATIC_CACHE += stocks
funds = investpy.get_funds_dict()
for x in stocks:
x['type'] = 'Fund'
__LOCAL_STATIC_CACHE += funds
# no country
all_items = __LOCAL_STATIC_CACHE
if isin:
all_items = [x for x in all_items if x['isin'] == isin]
if currency:
all_items = [x for x in all_items if x['currency'] == currency]
if all_items:
return all_items[0]
return None
def get_last_etf(keyword, country='united states'):
metal = ['Gold', 'Silver', 'Copper', 'Metal']
etf = pd.DataFrame(investpy.get_etfs_dict())
if keyword == 'Energy':
temp = etf[etf.full_name.str.contains(keyword)]
temp = temp.append(etf[etf.full_name.str.contains('Oil')])
elif keyword == 'Metal':
temp = pd.DataFrame()
for i in metal:
temp = temp.append(etf[etf.full_name.str.contains(i)])
temp.drop_duplicates(inplace=True)
elif keyword == 'commodity':
temp = etf[etf.asset_class == keyword]
metal.append('Oil')
for i in metal:
temp = temp.append(etf[~etf.full_name.str.contains(i)])
else:
temp = etf[etf.asset_class == keyword]
temp = temp[temp.country == country]
history = pd.DataFrame()
for i in temp.name.values:
try:
print(i)
data = investpy.get_etf_recent_data(
i, country=country)
data.reset_index(inplace=True)
data['name'] = i
history = history.append(data.tail(1))
time.sleep(random.choice(list(range(5, 16))))
except:
pass
return history
def test_etfs_errors():
"""
This function raises errors on etf retrieval functions
"""
try:
retrieve_etfs(test_mode=None)
except:
pass
params = [
{
'country': ['error']
},
{
'country': 'error'
},
]
for param in params:
try:
investpy.get_etfs(country=param['country'])
except:
pass
try:
investpy.get_etfs_list(country=param['country'])
except:
pass
params = [
{
'country': ['error'],
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': 'error'
},
{
'country': 'spain',
'columns': 0,
'as_json': True
},
{
'country': 'spain',
'columns': ['error'],
'as_json': False
},
]
for param in params:
try:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
except:
pass
params = [
{
'etf': None,
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': None,
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'netherlands',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': ['error'],
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True,
'order': 'error',
'debug': True
},
{
'etf': 'error',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_etf_recent_data(etf=param['etf'],
country=param['country'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'etf': None,
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'error',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'netherlands',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': None,
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': ['error'],
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'error',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': 'error',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'error',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': ['error'],
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/1998',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/01/1998',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/1900',
'to_date': '01/01/1950',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/03/2019',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_etf_historical_data(etf=param['etf'],
country=param['country'],
from_date=param['from_date'],
to_date=param['to_date'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'country': 'error',
'as_json': False,
},
{
'country': None,
'as_json': False,
},
{
'country': ['error'],
'as_json': False,
},
{
'country': 'spain',
'as_json': None,
},
{
'country': 'spain',
'as_json': ['error'],
},
]
for param in params:
try:
investpy.get_etfs_overview(country=param['country'])
except:
pass
params = [
{
'by': None,
'value': 'bbva',
},
{
'by': ['error'],
'value': 'bbva',
},
{
'by': 'error',
'value': 'bbva',
},
{
'by': 'name',
'value': None,
},
{
'by': 'name',
'value': ['error'],
},
{
'by': 'isin',
'value': 'error',
},
]
for param in params:
try:
investpy.search_etfs(by=param['by'], value=param['value'])
except:
pass
def test_investpy_etfs():
"""
This function checks that etf data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_etfs(country=param['country'])
investpy.get_etfs_list(country=param['country'])
params = [
{
'country': None,
'columns': ['name'],
'as_json': True
},
{
'country': None,
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': ['name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_etf_countries()
params = [
{
'as_json': True,
'order': 'ascending',
},
{
'as_json': False,
'order': 'ascending',
},
{
'as_json': True,
'order': 'descending',
},
{
'as_json': False,
'order': 'descending',
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
params = [
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': False
},
{
'etf': 'bbva accion dj eurostoxx 50',
'country': 'spain',
'as_json': True
}
]
for param in params:
investpy.get_etf_information(etf=param['etf'], country=param['country'], as_json=param['as_json'])
params = [
{
'country': 'united states',
'as_json': True,
'n_results': 2
},
{
'country': 'united kingdom',
'as_json': False,
'n_results': 2
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'], as_json=param['as_json'], n_results=param['n_results'])
investpy.search_etfs(by='name', value='bbva')
def test_investpy_etfs():
"""
This function checks that etf data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_etfs(country=param['country'])
investpy.get_etfs_list(country=param['country'])
params = [
{
'country': None,
'columns': ['id', 'name'],
'as_json': True
},
{
'country': None,
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_etf_countries()
params = [
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
params = [
{
'country': 'france',
'as_json': True,
},
{
'country': 'france',
'as_json': False,
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'],
as_json=param['as_json'])
investpy.search_etfs(by='name', value='bbva')
def test_investpy():
"""
This function checks that main functions of investpy work properly.
"""
print(investpy.__author__, investpy.__version__)
for value in ['spain', None]:
investpy.get_equities(country=value)
investpy.get_equities_list(country=value)
params = [
{
'country': None,
'columns': ['id', 'name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': True
},
{
'country': None,
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': False
},
]
for param in params:
investpy.get_equities_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_equity_countries()
params = [
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_recent_data(equity='enagás',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_historical_data(equity='enagás',
country='spain',
from_date='01/01/1990',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
for value in ['spanish', 'english']:
investpy.get_equity_company_profile(equity='enagás',
country='spain',
language=value)
retrieve_equities(test_mode=True)
retrieve_equity_countries(test_mode=True)
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_funds(country=param['country'])
investpy.get_funds_list(country=param['country'])
params = [
{
'country': None,
'columns': ['id', 'name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': True
},
{
'country': None,
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': False
},
]
for param in params:
investpy.get_funds_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
params = [
{
'fund': 'bbva multiactivo conservador pp',
'country': 'spain',
'as_json': True,
},
{
'fund': 'bbva multiactivo conservador pp',
'country': 'spain',
'as_json': False,
},
]
for param in params:
investpy.get_fund_information(fund=param['fund'],
country=param['country'],
as_json=param['as_json'])
params = [
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_fund_recent_data(fund='bbva multiactivo conservador pp',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_fund_historical_data(
fund='bbva multiactivo conservador pp',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_fund_countries()
investpy.get_funds()
retrieve_funds(test_mode=True)
retrieve_fund_countries(test_mode=True)
investpy.get_etf_countries()
for value in ['spain', None]:
investpy.get_etfs(country=value)
investpy.get_etfs_list(country=value)
params = [
{
'country': None,
'columns': ['id', 'name'],
'as_json': True
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': True
},
{
'country': None,
'columns': ['id', 'name'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': ['id', 'name'],
'as_json': False
},
]
for param in params:
investpy.get_etfs_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
params = [
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
country='spain',
from_date='01/01/2010',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
params = [
{
'country': 'france',
'as_json': True,
},
{
'country': 'usa',
'as_json': False,
},
]
for param in params:
investpy.get_etfs_overview(country=param['country'],
as_json=param['as_json'])
retrieve_etfs(test_mode=True)
def test_etf_errors():
params = [
{
'columns': None,
'as_json': 'error'
},
{
'columns': 0,
'as_json': True
},
{
'columns': ['error'],
'as_json': False
},
]
for param in params:
try:
investpy.get_etfs_dict(columns=param['columns'],
as_json=param['as_json'])
except:
pass
params = [
{
'etf': 'bbva accion dj eurostoxx 50',
'as_json': 'error',
'order': 'ascending'
},
{
'etf': 'bbva accion dj eurostoxx 50',
'as_json': True,
'order': 'error'
},
{
'etf': 'error',
'as_json': True,
'order': 'ascending'
},
]
for param in params:
try:
investpy.get_etf_recent_data(etf=param['etf'],
as_json=param['as_json'],
order=param['order'])
except:
pass
params = [
{
'etf': 'bbva accion dj eurostoxx 50',
'start': '01/01/2019',
'end': '01/01/2019',
'as_json': 'error',
'order': 'ascending'
},
{
'etf': 'bbva accion dj eurostoxx 50',
'start': '01/01/2019',
'end': '01/01/2019',
'as_json': False,
'order': 'error'
},
{
'etf': 'bbva accion dj eurostoxx 50',
'start': 'error',
'end': '01/01/2019',
'as_json': False,
'order': 'ascending'
},
{
'etf': 'bbva accion dj eurostoxx 50',
'start': '01/01/2019',
'end': 'error',
'as_json': False,
'order': 'ascending'
},
{
'etf': 'error',
'start': '01/01/2019',
'end': '01/01/2019',
'as_json': False,
'order': 'ascending'
},
{
'etf': 'bbva accion dj eurostoxx 50',
'start': '01/01/1998',
'end': '01/01/2019',
'as_json': False,
'order': 'ascending'
},
]
for param in params:
try:
investpy.get_etf_historical_data(etf=param['etf'],
start=param['start'],
end=param['end'],
as_json=param['as_json'],
order=param['order'])
except:
pass
def test_investpy():
"""
This function checks that main functions of investpy work properly.
"""
investpy.get_equities()
investpy.get_equities_list()
params = [
{
'as_json': False,
'order': 'ascending'
},
{
'as_json': True,
'order': 'descending'
},
{
'as_json': True,
'order': 'ascending'
},
{
'as_json': False,
'order': 'descending'
},
]
for param in params:
investpy.get_recent_data(equity='enagás',
as_json=param['as_json'],
order=param['order'])
investpy.get_historical_data(equity='enagás',
start='01/01/1990',
end='01/01/2019',
as_json=param['as_json'],
order=param['order'])
for value in ['spanish', 'english']:
investpy.get_equity_company_profile(equity='enagás', language=value)
get_equity_names()
investpy.get_funds()
investpy.get_funds_list()
for value in [True, False]:
investpy.get_funds_dict(columns=['id', 'name'], as_json=value)
investpy.get_fund_information(fund='bbva multiactivo conservador pp',
as_json=value)
params = [
{
'as_json': False,
'order': 'ascending'
},
{
'as_json': True,
'order': 'descending'
},
{
'as_json': True,
'order': 'ascending'
},
{
'as_json': False,
'order': 'descending'
},
]
for param in params:
investpy.get_fund_recent_data(fund='bbva multiactivo conservador pp',
as_json=param['as_json'],
order=param['order'])
investpy.get_fund_historical_data(
fund='bbva multiactivo conservador pp',
start='01/01/2010',
end='01/01/2019',
as_json=param['as_json'],
order=param['order'])
get_fund_names()
investpy.get_etfs()
investpy.get_etfs_list()
for value in [True, False]:
investpy.get_etfs_dict(columns=['id', 'name'], as_json=value)
params = [
{
'as_json': False,
'order': 'ascending'
},
{
'as_json': True,
'order': 'descending'
},
{
'as_json': True,
'order': 'ascending'
},
{
'as_json': False,
'order': 'descending'
},
]
for param in params:
investpy.get_etf_recent_data(etf='bbva accion dj eurostoxx 50',
as_json=param['as_json'],
order=param['order'])
investpy.get_etf_historical_data(etf='bbva accion dj eurostoxx 50',
start='01/01/2010',
end='01/01/2019',
as_json=param['as_json'],
order=param['order'])
get_etf_names()
