def api_get_df_c():
'''API that returns full price dataframe in JSON for input symbol(s)'''
global dir_db
global tup_model
global date_str
global live_data
db = DataBase([], dir_db)
j_data = request.get_json()
ls_sym = json.loads(j_data)['ls_sym']
time_str = json.loads(j_data)['time_str']
target_profit = 0.011
target_loss = -0.031
try:
ls_df = []
for sym in ls_sym:
df = get_df_c(sym, date_str, live_data, db, target_profit,
target_loss)
df = df[df['datetime'].dt.strftime('%H%M') <= time_str]
df_proba = get_df_proba(df, tup_model)
if not df_proba.empty:
df = pd.merge(df,
df_proba[['sym', 'datetime', 'proba']],
how='left',
on=['sym', 'datetime'])
else:
df['proba'] = None
ls_df.append(df)
df_c = pd.concat(ls_df)
j_df_c = df_c.to_json(orient='split')
db.close()
return j_df_c
except Exception as e:
print(sym, type(e).__name__, e.args) #traceback.print_exc()
db.close()
return pd.DataFrame().to_json(orient='split')
def api_get_df_c():
'''API that returns full cooked price dataframe in JSON for input symbol(s)
Returns:
j_df_c (JSON of pandas.Dataframe)
'''
global tup_model
global date_str
global live_data
global target_profit
global target_loss
global buffer_seconds
global pause
pause = 1
time.sleep(2)
db = DataBase([], DIR_DB)
j_data = request.get_json()
ls_sym = json.loads(j_data)['ls_sym']
time_str = json.loads(j_data)['time_str']
try:
ls_df = []
for sym in ls_sym:
time.sleep(buffer_seconds)
df = get_df_c(sym, date_str, live_data, db, target_profit,
target_loss)
df = df[df['datetime'].dt.strftime('%H%M') <= time_str]
df_proba = get_df_proba(df, tup_model)
if not df_proba.empty:
df = pd.merge(df,
df_proba[['sym', 'datetime', 'proba']],
how='left',
on=['sym', 'datetime'])
else:
df['proba'] = None
ls_df.append(df)
df_c = pd.concat(ls_df)
j_df_c = df_c.to_json(orient='split')
except Exception as e:
print(ERROR_EXCEPTION_SYM.format(sym, type(e).__name__, e.args))
j_df_c = pd.DataFrame().to_json(orient='split')
db.close()
pause = 0
return j_df_c
def update_predictions():
'''Runs an iteration of model predictions on
selected symbols and saves output in database
'''
db = DataBase([], DIR_DB)
df_sym = get_df_sym_filter(db, LS_SEC, LS_IND)
c_error = collections.Counter()
ls_skip = []
while 1:
dt_error = {}
idx_not_skip = -df_sym['sym'].isin(ls_skip)
for i, tup in tqdm(df_sym[idx_not_skip].iterrows(),
total=df_sym[idx_not_skip].shape[0]):
while pause:
time.sleep(BUFFER_SECONDS)
sym = tup['sym']
try:
time.sleep(BUFFER_SECONDS)
df_c = get_df_c(sym, DATE_STR, LIVE_DATA, db, TARGET_PROFIT,
TARGET_LOSS)
df_proba = get_df_proba(df_c, tup_model)
if not df_proba.empty:
df_proba.to_sql('proba',
db.conn,
if_exists='append',
index=0)
except Exception as e:
dt_error[sym] = ERROR_EXCEPTION.format(
type(e).__name__, e) # traceback.print_exc()
c_error.update([sym])
if dt_error:
num_runs = df_sym.shape[0]
[
print(ERROR_SUMMARY.format(sym, dt_error[sym]))
for sym in dt_error
]
print(
ERROR_PCT.format(len(dt_error), num_runs,
len(dt_error) / num_runs))
ls_skip = [k for k, v in c_error.items()
if v > ERROR_THRESHOLD] # skip symbols with too many errors
print(MSG_SKIP.format(ls_skip))
def update_predictions():
'''Runs an iteration of model predictions on
selected symbols and saves output in database
'''
global dir_db
global tup_model
global j_df_proba
global date_str
global live_data
global sym_limit
global buffer_seconds
db = DataBase([], dir_db)
ls_df_proba = []
target_profit = 0.011
target_loss = -0.031
df_sym = get_df_sym_filter(db)
df_sym = df_sym.iloc[:sym_limit]
while 1:
for i, tup in df_sym.iterrows():
if i % 100 == 0: print(i, df_sym.shape[0])
sym = tup['sym']
try:
df_c = get_df_c(sym, date_str, live_data, db, target_profit,
target_loss)
df_proba = get_df_proba(df_c, tup_model)
if not df_proba.empty:
df_proba.to_sql('proba',
db.conn,
if_exists='append',
index=0)
except Exception as e:
print(sym, type(e).__name__, e.args) #traceback.print_exc()
print(
f'Update complete, waiting for {buffer_seconds} seconds till next update...'
)
time.sleep(buffer_seconds)
'prev_close',
'divergence',
]
# extract and transform
print(MSG_DATE_RANGE.format(LS_DATE_STR[0], LS_DATE_STR[-1]))
ls_df_t = []
for date_str in LS_DATE_STR:
dt_errors = {}
num_runs = 0
print(date_str)
df_sym = get_df_sym(db, date_str)
for i, tup in tqdm(df_sym.iterrows(), total=df_sym.shape[0]):
sym = tup['sym']
num_runs += 1
try:
df_c = get_df_c(sym, date_str, LIVE_DATA, db, TARGET_PROFIT,
TARGET_LOSS)
ls_df_t.append(df_c[df_c['divergence'] != ''][LS_COL])
except Exception as e:
dt_errors[sym] = ERROR_EXCEPTION.format(type(e).__name__, e)
if dt_errors:
[print(ERROR_SUMMARY.format(sym, dt_errors[sym])) for sym in dt_errors]
print(
ERROR_PCT.format(len(dt_errors), num_runs,
len(dt_errors) / num_runs))
if ls_df_t:
# save df_train
df_t = pd.concat(ls_df_t)
df_t = df_t.dropna()
print(df_t.info())
# df_train - Export
timestamp = datetime.datetime.now().strftime('%Y%m%d_%H%M')
live_data = 0
target_profit = 0.011
target_loss = -0.031
ls_date_str = get_ls_date_str_from_db('2020-12-14', '2022-01-01', db) #'2020-06-23'
print('Creating df_train for date range: {} to {}'.format(ls_date_str[0], ls_date_str[-1]))
# extract and transform
count, count_e = 0, 0
ls_df_t = []
dt_errors = {}
for date_str in ls_date_str:
print(date_str)
df_sym = get_df_sym(db, date_str)
for i, tup in tqdm(df_sym.iterrows(), total=df_sym.shape[0]):
count+=1
try:
df_c = get_df_c(tup['sym'], date_str, live_data, db, target_profit, target_loss)
ls_df_t.append(df_c[df_c['divergence']!=''])
except Exception as e:
dt_errors[tup['sym']] = f'Error: {type(e).__name__} {e}'
count_e+=1
print('Errors:', count_e, count, round(count_e/count, 3))
print(dt_errors)
# save df_train
if ls_df_t:
df_t = pd.concat(ls_df_t)
ls_col = [
'is_profit',
'rsi14',
'sma9_var',
'sma180_var',
'vwap_var',