def main():
parser = argparse.ArgumentParser(description='scrap yahoo earning')
parser.add_argument('-output_prefix', type=str, default='data_tickers/investing_', help='output file prefix')
args = parser.parse_args()
# Index
indices = investpy.get_indices()
df_index = pd.DataFrame(indices)
df_index.to_csv(args.output_prefix + 'index_info.csv')
# Commodity
commodities = investpy.get_commodities()
commodities_dict = investpy.get_commodities_dict()
commodity_info_list = []
for commodity,country in zip(commodities['name'].to_list(),commodities['country'].to_list()):
print(commodity,',',country)
commodity_info_list.append(investpy.get_commodity_information(commodity,country))
time.sleep(scrap_delay)
df_commodity = pd.concat(commodity_info_list)
df_commodity.to_csv(args.output_prefix + 'commodity_info.csv')
# ETF
etfs_dict_us = investpy.get_etfs_dict('united states')
etfs_dict_canada = investpy.get_etfs_dict('canada')
df_etfs = pd.DataFrame(etfs_dict_us + etfs_dict_canada)
df_etfs.to_csv(args.output_prefix + 'etfs_info.csv')
# Stock
stocks_dict_us = investpy.get_stocks_dict('united states')
stocks_dict_canada = investpy.get_stocks_dict('canada')
df_stocks = pd.DataFrame(stocks_dict_us + stocks_dict_canada)
df_stocks.to_csv(args.output_prefix + 'stock_info.csv')
def _build_index(self):
def with_index(df, type, column, new_name="symbol"):
if not "country" in df:
df["country"] = "unknown"
else:
df["country"] = df["country"].replace({
None: "unknown",
"": "unknown"
}).fillna('unknown')
df.index = pd.MultiIndex.from_tuples([
(s, type, c)
for s, c in zip(df[column].to_list(), df["country"].to_list())
]).rename([
new_name if new_name is not None else column, "type", "country"
])
df = df.drop([column, "country"], axis=1)
df = df.drop(df.index[df.index.duplicated('first')], axis=0)
return df.loc[df.index.dropna()]
symbols_df = pd.concat(
[
with_index(ip.get_bonds(), "BOND",
"name"), # country "name" full_name
with_index(
ip.get_certificates(), "CERT", "symbol"
), # country', 'name', 'full_name', '"symbol"', 'issuer', 'isin', 'asset_class', 'underlying'
with_index(ip.get_cryptos(), "CRYPTO",
"symbol"), # 'name', '"symbol"', 'currency'
with_index(
ip.get_commodities(), "COMM", "name"
), # 'title', 'country', '"name"', 'full_name', 'currency', 'group'
with_index(
ip.get_etfs(), "ETF", "symbol"
), # 'country', 'name', 'full_name', '"symbol"', 'isin', 'asset_class', 'currency', 'stock_exchange', 'def_stock_exchange'
# with_index(ip.get_funds(), "FUND", "isin"), # 'country', 'name', 'symbol', 'issuer', '"isin"', 'asset_class', 'currency', 'underlying'
with_index(
ip.get_indices(), "INDEX", "symbol"
), # 'country', 'name', 'full_name', '"symbol"', 'currency', 'class', 'market'
with_index(
ip.get_stocks(), "STOCK", "symbol"
), # ['country', 'name', 'full_name', 'isin', 'currency', '"symbol"'
with_index(
pd.DataFrame(
[f'{c}/USD' for c in ip.get_available_currencies()],
columns=['symbol']), "FX", "symbol")
],
axis=0)
# update the index table
upsert(self.engine,
symbols_df,
DataProvider.symbols_table_name,
if_row_exists='ignore')
def test_investpy_commodities():
"""
This function checks that commodity data retrieval functions listed in investpy work properly.
"""
params = [
{
'group': 'metals',
},
{
'group': None,
},
]
for param in params:
investpy.get_commodities(group=param['group'])
investpy.get_commodities_list(group=param['group'])
params = [
{
'group': None,
'columns': ['title', 'full_name', 'name'],
'as_json': True
},
{
'group': None,
'columns': ['title', 'full_name', 'name'],
'as_json': False
},
{
'group': 'metals',
'columns': ['title', 'full_name', 'name'],
'as_json': True
},
{
'group': 'metals',
'columns': ['title', 'full_name', 'name'],
'as_json': False
},
{
'group': 'metals',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_commodities_dict(group=param['group'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_commodity_groups()
params = [
{
'country': None,
'as_json': True,
'order': 'ascending',
},
{
'country': 'united states',
'as_json': False,
'order': 'ascending',
},
{
'country': 'united states',
'as_json': True,
'order': 'descending',
},
{
'country': 'united states',
'as_json': False,
'order': 'descending',
},
]
for param in params:
investpy.get_commodity_recent_data(commodity='copper',
country=param['country'],
as_json=param['as_json'],
order=param['order'],
interval='Daily')
investpy.get_commodity_historical_data(commodity='copper',
from_date='01/01/1990',
to_date='01/01/2019',
country=param['country'],
as_json=param['as_json'],
order=param['order'],
interval='Daily')
params = [
{
'commodity': 'copper',
'country': None,
'as_json': False
},
{
'commodity': 'copper',
'country': 'united states',
'as_json': True
}
]
for param in params:
investpy.get_commodity_information(commodity=param['commodity'], country=param['country'], as_json=param['as_json'])
params = [
{
'group': 'metals',
'as_json': True,
'n_results': 100
},
{
'group': 'metals',
'as_json': False,
'n_results': 100
}
]
for param in params:
investpy.get_commodities_overview(group=param['group'], as_json=param['as_json'], n_results=param['n_results'])
investpy.search_commodities(by='name', value='gold')
