def main(): parser = argparse.ArgumentParser(description='scrap yahoo earning') parser.add_argument('-output_prefix', type=str, default='data_tickers/investing_', help='output file prefix') args = parser.parse_args() # Index indices = investpy.get_indices() df_index = pd.DataFrame(indices) df_index.to_csv(args.output_prefix + 'index_info.csv') # Commodity commodities = investpy.get_commodities() commodities_dict = investpy.get_commodities_dict() commodity_info_list = [] for commodity,country in zip(commodities['name'].to_list(),commodities['country'].to_list()): print(commodity,',',country) commodity_info_list.append(investpy.get_commodity_information(commodity,country)) time.sleep(scrap_delay) df_commodity = pd.concat(commodity_info_list) df_commodity.to_csv(args.output_prefix + 'commodity_info.csv') # ETF etfs_dict_us = investpy.get_etfs_dict('united states') etfs_dict_canada = investpy.get_etfs_dict('canada') df_etfs = pd.DataFrame(etfs_dict_us + etfs_dict_canada) df_etfs.to_csv(args.output_prefix + 'etfs_info.csv') # Stock stocks_dict_us = investpy.get_stocks_dict('united states') stocks_dict_canada = investpy.get_stocks_dict('canada') df_stocks = pd.DataFrame(stocks_dict_us + stocks_dict_canada) df_stocks.to_csv(args.output_prefix + 'stock_info.csv')
def _build_index(self): def with_index(df, type, column, new_name="symbol"): if not "country" in df: df["country"] = "unknown" else: df["country"] = df["country"].replace({ None: "unknown", "": "unknown" }).fillna('unknown') df.index = pd.MultiIndex.from_tuples([ (s, type, c) for s, c in zip(df[column].to_list(), df["country"].to_list()) ]).rename([ new_name if new_name is not None else column, "type", "country" ]) df = df.drop([column, "country"], axis=1) df = df.drop(df.index[df.index.duplicated('first')], axis=0) return df.loc[df.index.dropna()] symbols_df = pd.concat( [ with_index(ip.get_bonds(), "BOND", "name"), # country "name" full_name with_index( ip.get_certificates(), "CERT", "symbol" ), # country', 'name', 'full_name', '"symbol"', 'issuer', 'isin', 'asset_class', 'underlying' with_index(ip.get_cryptos(), "CRYPTO", "symbol"), # 'name', '"symbol"', 'currency' with_index( ip.get_commodities(), "COMM", "name" ), # 'title', 'country', '"name"', 'full_name', 'currency', 'group' with_index( ip.get_etfs(), "ETF", "symbol" ), # 'country', 'name', 'full_name', '"symbol"', 'isin', 'asset_class', 'currency', 'stock_exchange', 'def_stock_exchange' # with_index(ip.get_funds(), "FUND", "isin"), # 'country', 'name', 'symbol', 'issuer', '"isin"', 'asset_class', 'currency', 'underlying' with_index( ip.get_indices(), "INDEX", "symbol" ), # 'country', 'name', 'full_name', '"symbol"', 'currency', 'class', 'market' with_index( ip.get_stocks(), "STOCK", "symbol" ), # ['country', 'name', 'full_name', 'isin', 'currency', '"symbol"' with_index( pd.DataFrame( [f'{c}/USD' for c in ip.get_available_currencies()], columns=['symbol']), "FX", "symbol") ], axis=0) # update the index table upsert(self.engine, symbols_df, DataProvider.symbols_table_name, if_row_exists='ignore')
def test_investpy_commodities(): """ This function checks that commodity data retrieval functions listed in investpy work properly. """ params = [ { 'group': 'metals', }, { 'group': None, }, ] for param in params: investpy.get_commodities(group=param['group']) investpy.get_commodities_list(group=param['group']) params = [ { 'group': None, 'columns': ['title', 'full_name', 'name'], 'as_json': True }, { 'group': None, 'columns': ['title', 'full_name', 'name'], 'as_json': False }, { 'group': 'metals', 'columns': ['title', 'full_name', 'name'], 'as_json': True }, { 'group': 'metals', 'columns': ['title', 'full_name', 'name'], 'as_json': False }, { 'group': 'metals', 'columns': None, 'as_json': False }, ] for param in params: investpy.get_commodities_dict(group=param['group'], columns=param['columns'], as_json=param['as_json']) investpy.get_commodity_groups() params = [ { 'country': None, 'as_json': True, 'order': 'ascending', }, { 'country': 'united states', 'as_json': False, 'order': 'ascending', }, { 'country': 'united states', 'as_json': True, 'order': 'descending', }, { 'country': 'united states', 'as_json': False, 'order': 'descending', }, ] for param in params: investpy.get_commodity_recent_data(commodity='copper', country=param['country'], as_json=param['as_json'], order=param['order'], interval='Daily') investpy.get_commodity_historical_data(commodity='copper', from_date='01/01/1990', to_date='01/01/2019', country=param['country'], as_json=param['as_json'], order=param['order'], interval='Daily') params = [ { 'commodity': 'copper', 'country': None, 'as_json': False }, { 'commodity': 'copper', 'country': 'united states', 'as_json': True } ] for param in params: investpy.get_commodity_information(commodity=param['commodity'], country=param['country'], as_json=param['as_json']) params = [ { 'group': 'metals', 'as_json': True, 'n_results': 100 }, { 'group': 'metals', 'as_json': False, 'n_results': 100 } ] for param in params: investpy.get_commodities_overview(group=param['group'], as_json=param['as_json'], n_results=param['n_results']) investpy.search_commodities(by='name', value='gold')