def import_commodities(successfulPulls): # imports commodities from around the world in USD search_results = investpy.search_commodities(by='currency', value='USD') list_of_commodity_names = search_results["name"] firstIndex = datetime.datetime.strptime(_configKeys.STARTPULL, '%d/%m/%Y') lastIndex = datetime.datetime.strptime(_configKeys.ENDPULL, '%d/%m/%Y') for name in list_of_commodity_names[:2500]: try: # Have an if statement in place in case if we don't want to pull every etf because there are a lot of stocks # Program takes a long time to run if we have to webscrape every etf each time we run commodityData = [] commodityData = investpy.get_commodity_historical_data( commodity=name, country='united states', from_date=_configKeys.STARTPULL, to_date=_configKeys.ENDPULL) newIndex = [] for index in commodityData.index: newIndex.append( datetime.datetime.strptime( datetime.datetime.strftime((index + timedelta(days=1)), '%Y-%m-%d'), '%Y-%m-%d')) commodityData['Date'] = newIndex commodityData.set_index('Date', inplace=True) # If there's something that's been loaded into stockData, then the length is no longer 0 # if the differences is under 2~3 days, then it is ok to take this data since there is still enough data in the week to be usable # this timedelta fixes the problem of trying to pull during a long weekend name = str(name) + "Commodity" if commodityData.empty == False and commodityData.index[ 0] - firstIndex <= timedelta( days=2 ) and lastIndex - commodityData.index[-1] <= timedelta( days=3): successfulPulls["Symbol"].append(name) successfulPulls["Type"].append("Commodity") commodityData.to_csv( os.path.join(Path(_configKeys.DATA_FOLDER), name + '.csv')) except: print("Something went wrong when importing: " + name)
def test_investpy_commodities(): """ This function checks that commodity data retrieval functions listed in investpy work properly. """ params = [ { 'group': 'metals', }, { 'group': None, }, ] for param in params: investpy.get_commodities(group=param['group']) investpy.get_commodities_list(group=param['group']) params = [ { 'group': None, 'columns': ['title', 'full_name', 'name'], 'as_json': True }, { 'group': None, 'columns': ['title', 'full_name', 'name'], 'as_json': False }, { 'group': 'metals', 'columns': ['title', 'full_name', 'name'], 'as_json': True }, { 'group': 'metals', 'columns': ['title', 'full_name', 'name'], 'as_json': False }, { 'group': 'metals', 'columns': None, 'as_json': False }, ] for param in params: investpy.get_commodities_dict(group=param['group'], columns=param['columns'], as_json=param['as_json']) investpy.get_commodity_groups() params = [ { 'country': None, 'as_json': True, 'order': 'ascending', }, { 'country': 'united states', 'as_json': False, 'order': 'ascending', }, { 'country': 'united states', 'as_json': True, 'order': 'descending', }, { 'country': 'united states', 'as_json': False, 'order': 'descending', }, ] for param in params: investpy.get_commodity_recent_data(commodity='copper', country=param['country'], as_json=param['as_json'], order=param['order'], interval='Daily') investpy.get_commodity_historical_data(commodity='copper', from_date='01/01/1990', to_date='01/01/2019', country=param['country'], as_json=param['as_json'], order=param['order'], interval='Daily') params = [ { 'commodity': 'copper', 'country': None, 'as_json': False }, { 'commodity': 'copper', 'country': 'united states', 'as_json': True } ] for param in params: investpy.get_commodity_information(commodity=param['commodity'], country=param['country'], as_json=param['as_json']) params = [ { 'group': 'metals', 'as_json': True, 'n_results': 100 }, { 'group': 'metals', 'as_json': False, 'n_results': 100 } ] for param in params: investpy.get_commodities_overview(group=param['group'], as_json=param['as_json'], n_results=param['n_results']) investpy.search_commodities(by='name', value='gold')