def load_key(index=0):
"""!
Load a key in the list by index
"""
print(root_dir)
key_list = import_object(root_dir / "KEYS")
if key_list:
print("Found available keys:", key_list)
return key_list[index]
else:
return None
def remove_key(key):
key_list = import_object(root_dir / "KEYS")
if key_list:
if key in key_list:
key_list.remove(key)
flag = export_object(root_dir / "KEYS", key_list)
else:
print("Key", key, "not found.")
flag = 0
else:
flag = 1
return flag
def generate_adjclose_df(ticker_file="ETFTickers"):
"""
Generate a dataframe whose columns are:
date, symbol_1, symbol_2, symbol_3, etc...
The values in the symbol columns are the adjusted close values of that date.
TODO: fix this path stuff
"""
print("Generating the master adjusted close dataframe...")
tickers = import_object((root_dir / ticker_file).with_suffix('.pickle'))
main_df = pd.DataFrame()
for _, ticker in enumerate(tickers):
if ticker_file == "./tickers/sp500tickers.pickle":
datafile = './sp500_dfs/{}.csv'.format(ticker[1])
elif ticker_file == "./tickers/ETFTickers.pickle":
datafile = './ETF_dfs/{}.csv'.format(ticker[1])
else:
print("Unrecognized ticker file:", ticker_file)
if os.path.isfile(datafile):
print("Processing", ticker[1])
df = pd.read_csv(datafile, engine='python')
df.set_index('date', inplace=True)
df.rename(columns={'5. adjusted close': ticker[1]}, inplace=True)
df.drop(['1. open', '2. high', '3. low', '4. close', '6. volume',
'7. dividend amount', '8. split coefficient'], 1, inplace=True)
if main_df.empty:
main_df = df
else:
main_df = main_df.join(df, how='outer')
main_df = main_df.reindex(sorted(main_df.columns), axis=1)
print("Master adjusted close dataframe generated.")
if ticker_file == "./tickers/sp500tickers.pickle":
main_df.to_csv('sp500_bysymbol_joined_closes.csv')
elif ticker_file == "./tickers/ETFTickers.pickle":
main_df.to_csv('ETF_bysymbol_joined_closes.csv')
else:
print("Unrecognized ticker file:", ticker_file)
return main_df
def add_key(key):
"""!
Add a new key to the list.
"""
key_list = import_object(root_dir / "KEYS")
if key_list:
if key in key_list:
print("Key", key, "already stored.")
flag = 0
else:
key_list.append(key)
flag = export_object(root_dir / "KEYS", key_list)
else:
print("Generating new key list.")
key_list = list()
key_list.append(key)
flag = export_object(root_dir / "KEYS", key_list)
return flag
def get_data_from_alphaVantage(key,
reload=False,
ticker_file="ETFTickers",
symbol_market="TSX",
output_size="full"):
root_dir = Path(__file__).resolve().parent.parent
if reload:
tickers = obtain_tickers(ticker_file)
else:
tickers = import_object(
(root_dir / ticker_file).with_suffix('.pickle'))
folder_name = root_dir / 'dataframes' / (ticker_file.split("Tickers")[0])
folder_name.mkdir(parents=True, exist_ok=True)
ts = TimeSeries(key=key, output_format='pandas')
# If today is a weekend, go back to the friday
today = datetime.datetime.today()
weekday = today.weekday()
if weekday == 5:
today = datetime.datetime.today() - datetime.timedelta(days=1)
elif weekday == 6:
today = datetime.datetime.today() - datetime.timedelta(days=2)
else:
pass
today = today.strftime("%Y-%m-%d")
totalAPICalls = 0
maxAPICalls = 500 # Max of 500 API requests per day on free license
api_calls_per_minute = 5
sleep_delay = int(60 / api_calls_per_minute) + 2
for ticker in tickers:
# Check if a dataframe for the ticker already exists
ticker_file = (folder_name / ticker[1]).with_suffix('.csv')
if Path(ticker_file).is_file():
df = pd.read_csv(ticker_file)
df.set_index('date', inplace=True)
if today in df.index: # Check if ticker information is up to date
print(ticker[1], "is up to date.")
continue
if totalAPICalls < maxAPICalls:
if symbol_market == "TSX":
tickername = 'TSX:' + ticker[1]
try:
data, _ = ts.get_daily_adjusted(
tickername, outputsize=output_size) # full or compact
totalAPICalls = totalAPICalls + 1
print('Retrieved data for:', tickername, '(', ticker[0],
')')
got_flag = True
except Exception as e:
print("Error retrieving data for", tickername + ":", e)
if output_size == "full":
try:
tickername = 'TSX:' + ticker[1]
data, _ = ts.get_daily_adjusted(
tickername, outputsize='compact')
totalAPICalls = totalAPICalls + 1
print('Retrieved compact data for:', tickername,
'(', ticker[0], ')')
got_flag = True
except Exception as e:
print("Error retrieving data for",
tickername + ":", e)
got_flag = False
time.sleep(sleep_delay)
else:
time.sleep(sleep_delay)
got_flag = False
else:
try:
data, _ = ts.get_daily_adjusted(ticker[1],
outputsize=output_size)
totalAPICalls = totalAPICalls + 1
print('Retrieved data for:', ticker[1], '(', ticker[0],
')')
got_flag = True
except Exception as e:
print("Error retrieving data for", ticker[1] + ":", e)
time.sleep(sleep_delay)
got_flag = False
if got_flag:
if Path(ticker_file).is_file():
# if os.path.exists('{}/{}.csv'.format(folder_name, ticker[1])):
data.to_csv('temp.csv')
df_new = pd.read_csv('temp.csv',
parse_dates=True,
index_col=0)
df_old = pd.read_csv(ticker_file,
parse_dates=True,
index_col=0)
# Merge and drop exact duplicates
df = pd.concat([df_new, df_old]).drop_duplicates()
# Drops duplicates with updated values, keeping the most recent data
df = df.loc[~df.index.duplicated(keep='first')]
else:
data.to_csv(ticker_file)
df = pd.read_csv(ticker_file,
parse_dates=True,
index_col=0)
df.sort_index(axis=0, inplace=True)
df = cleanup_zeros(df)
df.to_csv(ticker_file)
time.sleep(sleep_delay)
else:
print("Already used 500 API requests. Have to wait 24 hours now.")