def main():
parser = argparse.ArgumentParser(description='scrap yahoo earning')
parser.add_argument('-output_prefix', type=str, default='data_tickers/investing_', help='output file prefix')
args = parser.parse_args()
# Index
indices = investpy.get_indices()
df_index = pd.DataFrame(indices)
df_index.to_csv(args.output_prefix + 'index_info.csv')
# Commodity
commodities = investpy.get_commodities()
commodities_dict = investpy.get_commodities_dict()
commodity_info_list = []
for commodity,country in zip(commodities['name'].to_list(),commodities['country'].to_list()):
print(commodity,',',country)
commodity_info_list.append(investpy.get_commodity_information(commodity,country))
time.sleep(scrap_delay)
df_commodity = pd.concat(commodity_info_list)
df_commodity.to_csv(args.output_prefix + 'commodity_info.csv')
# ETF
etfs_dict_us = investpy.get_etfs_dict('united states')
etfs_dict_canada = investpy.get_etfs_dict('canada')
df_etfs = pd.DataFrame(etfs_dict_us + etfs_dict_canada)
df_etfs.to_csv(args.output_prefix + 'etfs_info.csv')
# Stock
stocks_dict_us = investpy.get_stocks_dict('united states')
stocks_dict_canada = investpy.get_stocks_dict('canada')
df_stocks = pd.DataFrame(stocks_dict_us + stocks_dict_canada)
df_stocks.to_csv(args.output_prefix + 'stock_info.csv')
def get_stock_lists():
'''Retrieve stock lists as dataframe tables, from investpy library'''
stocks = investpy.get_stocks()
etfs = investpy.get_etfs()
indices = investpy.get_indices()
return stocks, etfs, indices
def _build_index(self):
def with_index(df, type, column, new_name="symbol"):
if not "country" in df:
df["country"] = "unknown"
else:
df["country"] = df["country"].replace({
None: "unknown",
"": "unknown"
}).fillna('unknown')
df.index = pd.MultiIndex.from_tuples([
(s, type, c)
for s, c in zip(df[column].to_list(), df["country"].to_list())
]).rename([
new_name if new_name is not None else column, "type", "country"
])
df = df.drop([column, "country"], axis=1)
df = df.drop(df.index[df.index.duplicated('first')], axis=0)
return df.loc[df.index.dropna()]
symbols_df = pd.concat(
[
with_index(ip.get_bonds(), "BOND",
"name"), # country "name" full_name
with_index(
ip.get_certificates(), "CERT", "symbol"
), # country', 'name', 'full_name', '"symbol"', 'issuer', 'isin', 'asset_class', 'underlying'
with_index(ip.get_cryptos(), "CRYPTO",
"symbol"), # 'name', '"symbol"', 'currency'
with_index(
ip.get_commodities(), "COMM", "name"
), # 'title', 'country', '"name"', 'full_name', 'currency', 'group'
with_index(
ip.get_etfs(), "ETF", "symbol"
), # 'country', 'name', 'full_name', '"symbol"', 'isin', 'asset_class', 'currency', 'stock_exchange', 'def_stock_exchange'
# with_index(ip.get_funds(), "FUND", "isin"), # 'country', 'name', 'symbol', 'issuer', '"isin"', 'asset_class', 'currency', 'underlying'
with_index(
ip.get_indices(), "INDEX", "symbol"
), # 'country', 'name', 'full_name', '"symbol"', 'currency', 'class', 'market'
with_index(
ip.get_stocks(), "STOCK", "symbol"
), # ['country', 'name', 'full_name', 'isin', 'currency', '"symbol"'
with_index(
pd.DataFrame(
[f'{c}/USD' for c in ip.get_available_currencies()],
columns=['symbol']), "FX", "symbol")
],
axis=0)
# update the index table
upsert(self.engine,
symbols_df,
DataProvider.symbols_table_name,
if_row_exists='ignore')
def test_indices_errors():
"""
This function raises errors on index retrieval functions
"""
try:
retrieve_indices(test_mode=None)
except:
pass
try:
retrieve_index_countries(test_mode=None)
except:
pass
try:
retrieve_global_indices_countries(test_mode=None)
except:
pass
params = [
{
'country': ['error']
},
{
'country': 'error'
},
]
for param in params:
try:
investpy.get_indices(country=param['country'])
except:
pass
try:
investpy.get_indices_list(country=param['country'])
except:
pass
params = [
{
'country': ['error'],
'columns': None,
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': 'error'
},
{
'country': 'spain',
'columns': 0,
'as_json': True
},
{
'country': 'spain',
'columns': ['error'],
'as_json': False
},
]
for param in params:
try:
investpy.get_indices_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
except:
pass
params = [
{
'index': None,
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': ['error'],
'country': 'spain',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': None,
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'netherlands',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': ['error'],
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'as_json': True,
'order': 'error',
'debug': True
},
{
'index': 'error',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'index': ['error'],
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_index_recent_data(index=param['index'],
country=param['country'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'index': None,
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'error',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'netherlands',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': None,
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': ['error'],
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': 'error',
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'error',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': 'error',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': 'error',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'error',
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': ['error'],
'country': 'spain',
'from_date': '01/01/2018',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/1998',
'to_date': '01/01/2019',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/01/1998',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/1900',
'to_date': '01/01/1950',
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'index': 'ibex 35',
'country': 'spain',
'from_date': '01/01/2019',
'to_date': '01/03/2019',
'as_json': True,
'order': 'ascending',
'debug': 'error'
},
]
for param in params:
try:
investpy.get_index_historical_data(index=param['index'],
country=param['country'],
from_date=param['from_date'],
to_date=param['to_date'],
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
except:
pass
params = [
{
'by': None,
'value': 'ibex',
},
{
'by': ['error'],
'value': 'ibex',
},
{
'by': 'error',
'value': 'ibex',
},
{
'by': 'name',
'value': None,
},
{
'by': 'name',
'value': ['error'],
},
{
'by': 'name',
'value': 'error',
},
]
for param in params:
try:
investpy.search_indices(by=param['by'], value=param['value'])
except:
pass
def test_investpy_indices():
"""
This function checks that index data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_indices(country=param['country'])
investpy.get_indices_list(country=param['country'])
params = [
{
'country': None,
'columns': ['name', 'currency'],
'as_json': True
},
{
'country': None,
'columns': ['name', 'currency'],
'as_json': False
},
{
'country': 'spain',
'columns': ['name', 'currency'],
'as_json': True
},
{
'country': 'spain',
'columns': ['name', 'currency'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_indices_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_index_countries()
params = [
{
'as_json': True,
'order': 'ascending',
},
{
'as_json': False,
'order': 'ascending',
},
{
'as_json': True,
'order': 'descending',
},
{
'as_json': False,
'order': 'descending',
},
]
for param in params:
investpy.get_index_recent_data(index='ibex 35',
country='spain',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
investpy.get_index_historical_data(index='ibex 35',
country='spain',
from_date='01/01/2018',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
interval='Daily')
params = [
{
'index': 'ibex 35',
'country': 'spain',
'as_json': False
},
{
'index': 'ibex 35',
'country': 'spain',
'as_json': True
}
]
for param in params:
investpy.get_index_information(index=param['index'], country=param['country'], as_json=param['as_json'])
params = [
{
'country': 'united states',
'as_json': False,
'n_results': 10
},
{
'country': 'united kingdom',
'as_json': True,
'n_results': 10
}
]
for param in params:
investpy.get_indices_overview(country=param['country'], as_json=param['as_json'], n_results=param['n_results'])
investpy.search_indices(by='name', value='ibex')
def test_investpy_indices():
"""
This function checks that index data retrieval functions listed in investpy work properly.
"""
params = [
{
'country': 'spain',
},
{
'country': None,
},
]
for param in params:
investpy.get_indices(country=param['country'])
investpy.get_indices_list(country=param['country'])
params = [
{
'country': None,
'columns': ['name', 'currency'],
'as_json': True
},
{
'country': None,
'columns': ['name', 'currency'],
'as_json': False
},
{
'country': 'spain',
'columns': ['name', 'currency'],
'as_json': True
},
{
'country': 'spain',
'columns': ['name', 'currency'],
'as_json': False
},
{
'country': 'spain',
'columns': None,
'as_json': False
},
]
for param in params:
investpy.get_indices_dict(country=param['country'],
columns=param['columns'],
as_json=param['as_json'])
investpy.get_index_countries()
params = [
{
'as_json': True,
'order': 'ascending',
'debug': False
},
{
'as_json': False,
'order': 'ascending',
'debug': True
},
{
'as_json': True,
'order': 'descending',
'debug': False
},
{
'as_json': False,
'order': 'descending',
'debug': False
},
]
for param in params:
investpy.get_index_recent_data(index='ibex 35',
country='spain',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.get_index_historical_data(index='ibex 35',
country='spain',
from_date='01/01/2018',
to_date='01/01/2019',
as_json=param['as_json'],
order=param['order'],
debug=param['debug'])
investpy.search_indices(by='name', value='ibex')
# Created by Kim Hyeongjun on 05/01/2021.
# Copyright © 2021 dr-hkim.github.io. All rights reserved.
# investpy 패키지의 종목명은 investing.com 에서 검색 가능
import pandas as pd
import numpy as np
import investpy
import datetime as dt
# import matplotlib.pyplot as plt
# import mplfinance as mpf # 캔들차트
DD_END_DATE = "15/02/2022"
########################################################################################################################
investpy.get_indices(country=None)
list_index_us = investpy.get_indices(country="United States")
list_index_us_search = list_index_us[list_index_us['full_name'].str.contains(
"MSCI")]
list_index_kr = investpy.get_indices(country="south korea")
list_index_kr_search = list_index_kr[list_index_kr['full_name'].str.contains(
"MSCI")]
list_index_kr2 = list_index_kr.sort_values(by=['name'])
# 미국 S&P 500 지수 (1979.12.26 부터)
investpy_snp500 = investpy.get_index_historical_data(index="S&P 500",
country="United States",
from_date="30/01/1900",
to_date=DD_END_DATE)
investpy_snp500.to_pickle('./Market_Watch_Data/investpy_snp500.pkl')
index_sheet = 'indices'
stock_sheet = 'stocks'
bond_sheet = 'bonds'
etf_sheet = 'etfs'
crypto_sheet = 'cryptos'
home_sheet = 'Home'
fund_country = xw.sheets[home_sheet].range('F3').value
index_country = xw.sheets[home_sheet].range('F4').value
stock_country = xw.sheets[home_sheet].range('F5').value
bond_country = xw.sheets[home_sheet].range('F6').value
etf_country = xw.sheets[home_sheet].range('F7').value
req_funds = investpy.get_funds(country=fund_country)
req_indices = investpy.get_indices(country=index_country)
req_stocks = investpy.get_stocks(country=stock_country)
req_bonds = investpy.get_bonds(country=bond_country)
req_etfs = investpy.get_etfs(country=etf_country)
def GetFunds():
wb = xw.Book.caller()
wb.sheets[fund_sheet].range('D1').value = req_funds
def GetIndices():
wb = xw.Book.caller()
import investpy
import pymysql
while True:
with open("/password1.txt", "r") as f:
connection = pymysql.connect(host="fumire.moe",
user="******",
password=f.readline().strip(),
db="fumiremo_StockDB",
charset="utf8",
port=3306)
cursor = connection.cursor(pymysql.cursors.DictCursor)
for country in ["south korea", "japan", "united states"]:
print(country)
for _, row1 in investpy.get_indices(country=country).iterrows():
print("-", row1["name"])
try:
for date, row2 in investpy.get_index_historical_data(
index=row1["name"],
country=country,
from_date="01/01/1900",
to_date=datetime.datetime.today().strftime("%d/%m/%Y"),
order="descending").iterrows():
query = "SELECT * FROM `IndexData` WHERE `country` LIKE '%s' AND `Name` LIKE '%s' AND `Symbol` LIKE '%s' AND `Date` = '%s'" % (
country, row1["name"], row1["symbol"], date.date())
cursor.execute(query)
if cursor.fetchall():
break
# Retrieve all the available stocks as a Python list
investpy.get_stocks(country='brazil')
# In[70]:
# Retrieve all the available etfs as a Python list
investpy.get_etfs(country='brazil')
# In[138]:
# Retrieve all the indexes
investpy.get_indices()
# get data using investing website
indice = 'LBMA Gold Fixing Price'
country = 'united kingdom'
from_date = '19/11/2004'
to_date = '25/09/2020'
#investpy.indices.get_index_historical_data(indice, country, from_date, to_date, order='ascending', interval='Daily')
for item in investpy.search.search_quotes('OZ1'):
print(item)
# In[72]:
# get bonds
def my_get_indices(country=None):
return investpy.get_indices(country=country)
