def store_data_daily(backup_conn, ticker_list):
"""
Stores daily data for each ticker
:param conn: AWS psycopg2 connection
:param backup_conn: Backup psycopg2 connection
:param ticker_list: List of tickers for API calls
:param calls_per_minute: Max limit API calls per minute
"""
conn = db_wrapper.connect()
cur = conn.cursor()
columns = ('timestamp', 'open', 'high', 'low', 'close', 'volume',
'dividend_amount', 'split_coefficient')
for ticker in ticker_list:
table_name = 'data_daily_' + ticker.lower().replace('.', 'dot')
query = SQL(
'CREATE TABLE IF NOT EXISTS {} (timestamp date, open float(8), high float(8), low float(8), \
close float(8), volume float(8), dividend_amount float(8), split_coefficient float(8));'
)
success = False
BENCH.mark('API Call')
while not success:
try:
data_daily = api.get_daily(ticker, 'alpaca')
data_dividend, data_splits = api.get_actions_alpaca(ticker)
success = True
except (requests.exceptions.ConnectionError,
requests.exceptions.HTTPError):
print('API_ERROR: SLEEPING...')
sleep(1)
BENCH.mark('API Call')
connected = False
while not connected:
try:
store_data(conn, cur, backup_conn, columns, table_name, query,
data_daily)
connected = True
except psycopg2.OperationalError:
print('DB_ERROR: RECONNECTING...')
sleep(1)
conn = db_wrapper.connect()
cur = conn.cursor()
print('{message: <20}: Cached!\n'.format(message=table_name))
conn.close()
cur.close()
return
def ta_test():
ticker_list = [
'aapl', 'amzn', 'msft', 'amd', 'nvda', 'goog', 'baba', 'fitb', 'mu',
'fb', 'sq', 'tsm', 'qcom', 'mo', 'bp', 'unh', 'cvs', 'tpr'
]
data = {}
conn = connect()
timestamp1 = datetime(2008, 1, 1)
timestamp2 = datetime(2030, 1, 1)
for i in ticker_list:
data[i] = get_data_interval(conn,
'data_daily_{}'.format(i),
timestamp1,
timestamp2,
pandas=True)
data[i] = ta.add_all_ta_features(data[i],
open='open',
high='high',
low='low',
close='close',
volume='volume')
print(data[i])
train, test, label_train, label_test = process_data(data)
print_distribution(train, label_train)
print_distribution(test, label_test)
train_cnn(cnn(input_shape=(30, 4), num_classes=num_classes), train,
label_train, test, label_test)
def main():
conn = connect()
csv_daily_data(
conn, 'aapl', datetime(1998, 1, 2), datetime(2020, 2, 20),
'/mnt/c/Users/byron.LAPTOP-6A9A5QNU/Desktop/GitHub/algotrade/data')
csv_intraday_data(
conn, 'aapl', datetime(2020, 2, 14, 9, 31, 0),
datetime(2020, 2, 28, 16, 0, 0),
'/mnt/c/Users/byron.LAPTOP-6A9A5QNU/Desktop/GitHub/algotrade/data')
def ingest(environ, asset_db_writer, minute_bar_writer, daily_bar_writer,
adjustment_writer, calendar, start_session, end_session, cache,
show_progress, output_dir):
metadata_dtype = [('symbol', 'object'), ('asset_name', 'object'),
('start_date', 'datetime64[ns]'),
('end_date', 'datetime64[ns]'),
('first_traded', 'datetime64[ns]'),
('auto_close_date', 'datetime64[ns]'),
('exchange', 'object')]
ticker_list = [
'aapl', 'amzn', 'msft', 'amd', 'nvda', 'goog', 'baba', 'fitb',
'mu', 'fb', 'sq', 'tsm', 'qcom', 'mo', 'bp', 'unh', 'cvs', 'tpr'
]
conn = connect()
metadata = pd.DataFrame(
np.empty(len(ticker_list), dtype=metadata_dtype))
start_date = datetime(2020, 2, 14, 9, 31, 0)
end_date = datetime(2020, 2, 28, 16, 0, 0)
for i, ticker in enumerate(ticker_list):
intraday = get_data_interval(conn,
'data_intraday_{}'.format(ticker),
start_date, end_date)
daily = get_data_interval(conn, 'data_daily_{}'.format(ticker),
start_date.date(), end_date.date())
metadata.iloc[
i] = ticker, ticker, start_date, end_date, start_date, end_date + timedelta(
days=1), 'NYSE'
# print(metadata.iloc[i])
df = pd.DataFrame(
intraday,
columns=['date', 'open', 'high', 'low', 'close', 'volume'])
df = df.set_index('date')
df1 = pd.DataFrame(daily,
columns=[
'date', 'open', 'high', 'low', 'close',
'adjusted_close', 'volume', 'dividend',
'split'
])
df1 = df1.drop(['adjusted_close'], axis=1).set_index('date')
try:
minute_bar_writer.write([(i, df)], show_progress=True)
except Exception as e:
print(e)
try:
daily_bar_writer.write([(i, df1)], show_progress=True)
except Exception as e:
print(e)
asset_db_writer.write(equities=metadata)
print(metadata)
adjustment_writer.write()
def main():
# ticker_list = ['aapl', 'amzn', 'msft', 'amd', 'nvda', 'goog', 'baba', 'fitb', 'mu', 'fb', 'sq', 'tsm', 'qcom', 'mo',
# 'bp', 'unh', 'cvs', 'tpr']
ticker_list = ['aapl']
data = {}
conn = connect()
timestamp1 = datetime(2008, 1, 1)
timestamp2 = datetime(2030, 1, 1)
window = 50
num_classes = 4
dimensions = 1
label_window = 2
label_names = {0: 'losses', 1: 'gains', 2: 'both', 3: 'none'}
for i in ticker_list:
data[i] = get_data_interval(conn, 'data_daily_{}'.format(i),
timestamp1, timestamp2)
# train, test, label_train, label_test = process_cnn(data, window=window, label_window=label_window)
train, test, label_train, label_test = process_lstm(data, window=window)
# print_distribution(train, label_train, label_names)
# print_distribution(test, label_test, label_names)
# train_cnn(cnn(input_shape=(window, dimensions), num_classes=num_classes), train, label_train, test, label_test)
train_lstm(lstm(input_shape=(window, dimensions)), train, label_train,
test, label_test)