def get_futures_price_preloaded(**kwargs):
if 'ticker_head' in kwargs.keys():
ticker_head = kwargs['ticker_head']
else:
ticker = kwargs['ticker']
ticker_head = cmi.get_contract_specs(ticker)['ticker_head']
if 'futures_data_dictionary' in kwargs.keys():
data_out = kwargs['futures_data_dictionary'][ticker_head]
else:
presaved_futures_data_folder = dn.get_directory_name(ext='presaved_futures_data')
if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head + '.pkl'):
data_out = pd.read_pickle(presaved_futures_data_folder + '/' + ticker_head + '.pkl')
else:
data_out = pd.DataFrame()
return data_out
if 'settle_date' in kwargs.keys():
settle_date = kwargs['settle_date']
if isinstance(settle_date,int):
data_out = data_out[data_out['settle_date'] == cu.convert_doubledate_2datetime(settle_date)]
elif isinstance(settle_date,dt.datetime):
data_out = data_out[data_out['settle_date'] == settle_date]
if 'settle_date_from_exclusive' in kwargs.keys():
data_out = data_out[data_out['settle_date']>cu.convert_doubledate_2datetime(kwargs['settle_date_from_exclusive'])]
if 'ticker' in kwargs.keys():
data_out = data_out[data_out['ticker']==ticker]
return data_out
def get_futures_price_preloaded(**kwargs):
if 'ticker_head' in kwargs.keys():
ticker_head = kwargs['ticker_head']
else:
ticker = kwargs['ticker']
contract_specs_output = cmi.get_contract_specs(ticker)
ticker_head = contract_specs_output['ticker_head']
file_ticker = cmi.mini_contract_dictionary.get(
ticker_head,
ticker_head) + contract_specs_output['ticker_month_str'] + str(
contract_specs_output['ticker_year'])
if 'futures_data_dictionary' in kwargs.keys():
data_out = kwargs['futures_data_dictionary'][ticker_head]
else:
presaved_futures_data_folder = dn.get_directory_name(
ext='presaved_futures_data')
file_ticker_head = cmi.mini_contract_dictionary.get(
ticker_head, ticker_head)
if os.path.isfile(presaved_futures_data_folder + '/' +
file_ticker_head + '.pkl'):
data_out = pd.read_pickle(presaved_futures_data_folder + '/' +
file_ticker_head + '.pkl')
else:
data_out = pd.DataFrame()
return data_out
if 'settle_date' in kwargs.keys():
settle_date = kwargs['settle_date']
if isinstance(settle_date, int):
data_out = data_out[data_out['settle_date'] ==
cu.convert_doubledate_2datetime(settle_date)]
elif isinstance(settle_date, dt.datetime):
data_out = data_out[data_out['settle_date'] == settle_date]
if 'settle_date_from_exclusive' in kwargs.keys():
data_out = data_out[
data_out['settle_date'] > cu.convert_doubledate_2datetime(
kwargs['settle_date_from_exclusive'])]
if 'settle_date_from' in kwargs.keys():
data_out = data_out[
data_out['settle_date'] >= cu.convert_doubledate_2datetime(
kwargs['settle_date_from'])]
if 'settle_date_to' in kwargs.keys():
settle_date_to = kwargs['settle_date_to']
if isinstance(settle_date_to, int):
data_out = data_out[
data_out['settle_date'] <= cu.convert_doubledate_2datetime(
kwargs['settle_date_to'])]
elif isinstance(settle_date_to, dt.datetime):
data_out = data_out[data_out['settle_date'] <= settle_date_to]
if 'ticker' in kwargs.keys():
data_out = data_out[data_out['ticker'] == file_ticker]
return data_out
def get_tickers_from_agg_method_and_contracts_back(**kwargs):
ticker = kwargs['ticker']
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
contact_specs_out = cmi.get_contract_specs(ticker)
ref_date = 10000 * contact_specs_out[
'ticker_year'] + 100 * contact_specs_out['ticker_month_num'] + 1
ref_datetime = cu.convert_doubledate_2datetime(ref_date)
if aggregation_method == 12:
cal_date_list = [
ref_datetime - relativedelta(years=x)
for x in range(1, contracts_back + 1)
]
elif aggregation_method == 1:
cal_date_list = [
ref_datetime - relativedelta(months=x)
for x in range(1, contracts_back + 1)
]
ticker_list = [
contact_specs_out['ticker_head'] +
cmi.full_letter_month_list[x.month - 1] + str(x.year)
for x in cal_date_list
]
return ticker_list
def get_symbol_list(**kwargs):
quote_type = kwargs['quote_type']
datetime_from = cu.convert_doubledate_2datetime(20190317)
datetime_to = dt.datetime.combine(dt.date.today(), dt.datetime.min.time())
delta = datetime_to - datetime_from # as timedelta
ticker_list = []
for i in range(delta.days):
day = datetime_from + dt.timedelta(days=i)
ticker_frame_out = bgd.get_ticker_frame(date=int(day.strftime('%Y%m%d')))
ticker_list.extend(list(ticker_frame_out['symbol']))
ticker_list = list(set(ticker_list))
if quote_type == 'BTC':
select_indx = -3
elif quote_type == 'USDT':
select_indx = -4
res = [i for i in ticker_list if i[select_indx:] == quote_type]
return res
def doubledate_shift_bus_days(**kwargs):
if 'double_date' in kwargs.keys():
double_date = kwargs['double_date']
else:
double_date = int(tm.strftime('%Y%m%d'))
if 'shift_in_days' in kwargs.keys():
shift_in_days = kwargs['shift_in_days']
else:
shift_in_days = 1
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead = const.reference_tickerhead_4business_calendar
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
double_date_datetime = cu.convert_doubledate_2datetime(double_date)
if shift_in_days < 0:
dts_aux = pd.date_range(double_date_datetime, periods=-shift_in_days+1, freq=bday_us)
dts = [x for x in dts_aux if x.to_pydatetime() != double_date_datetime]
shifted_datetime = dts[-shift_in_days-1]
elif shift_in_days > 0:
dts_aux = pd.date_range(start=double_date_datetime-dt.timedelta(max(m.ceil(shift_in_days*7/4), shift_in_days+8)),
end=double_date_datetime, freq=bday_us)
dts = [x for x in dts_aux if x.to_pydatetime() != double_date_datetime]
shifted_datetime = dts[-shift_in_days]
return int(shifted_datetime.strftime('%Y%m%d'))
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs):
ref_date = kwargs['ref_date']
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
ref_datetime = cu.convert_doubledate_2datetime(ref_date)
if aggregation_method == 12:
cal_date_list = [ref_datetime - relativedelta(years=x) for x in range(1, contracts_back+1)]
elif aggregation_method == 1:
cal_date_list = [ref_datetime - relativedelta(months=x) for x in range(1, contracts_back+1)]
bday_us = CustomBusinessDay(expcalendar=exp.get_calendar_4ticker_head(const.reference_tickerhead_4business_calendar))
if 'shift_bus_days' in kwargs.keys():
shift_bus_days = kwargs['shift_bus_days']
if shift_bus_days >= 0:
bus_date_list = [pd.date_range(x, periods=shift_bus_days+1, freq=bday_us)[shift_bus_days].to_datetime() for x in cal_date_list]
elif shift_bus_days < 0:
bus_date_list = [pd.date_range(start=x-relativedelta(days=(max(m.ceil(-shift_bus_days*7/5)+5, -shift_bus_days+5))), end=x, freq=bday_us)[shift_bus_days-1].to_datetime() for x in cal_date_list]
else:
bus_date_list = [pd.date_range(x, periods=1, freq=bday_us)[0].to_datetime() for x in cal_date_list]
return bus_date_list
def get_days2_roll(**kwargs):
ticker = kwargs['ticker']
date_to = kwargs['date_to']
contract_specs = cmf.get_contract_specs(ticker)
ticker_head = contract_specs['ticker_head']
ticker_class = cmf.ticker_class[ticker_head]
ticker_year = contract_specs['ticker_year']
ticker_month_num = contract_specs['ticker_month_num']
datetime_to = cu.convert_doubledate_2datetime(date_to)
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(ticker_head))
if ticker_class == 'Metal':
if ticker_month_num == 1:
ticker_year_roll = ticker_year-1
ticker_month_roll = 12
else:
ticker_year_roll = ticker_year
ticker_month_roll = ticker_month_num-1
roll_datetime = dt.date(ticker_year_roll,ticker_month_roll,25)
else:
expiration_datetime = get_expiration_from_db(**kwargs)
roll_datetime = expiration_datetime-dt.timedelta(days=5)
if roll_datetime > datetime_to.date():
dts = pd.date_range(start=datetime_to, end=roll_datetime, freq=bday_us)
tr_days_2roll = len(dts)-1
else:
dts = pd.date_range(start=roll_datetime, end=datetime_to, freq=bday_us)
tr_days_2roll = -(len(dts)-1)
return {'roll_datetime': roll_datetime,
'cal_days_2roll': (roll_datetime-datetime_to.date()).days,
'tr_days_2roll': tr_days_2roll}
def doubledate_shift_bus_days(**kwargs):
if 'double_date' in kwargs.keys():
double_date = kwargs['double_date']
else:
double_date = int(tm.strftime('%Y%m%d'))
if 'shift_in_days' in kwargs.keys():
shift_in_days = kwargs['shift_in_days']
else:
shift_in_days = 1
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead = const.reference_tickerhead_4business_calendar
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
double_date_datetime = cu.convert_doubledate_2datetime(double_date)
if shift_in_days < 0:
dts_aux = pd.date_range(double_date_datetime, periods=-shift_in_days+1, freq=bday_us)
dts = [x for x in dts_aux if x.to_datetime() != double_date_datetime]
shifted_datetime = dts[-shift_in_days-1]
elif shift_in_days > 0:
dts_aux = pd.date_range(start=double_date_datetime-dt.timedelta(max(m.ceil(shift_in_days*7/4), shift_in_days+5)),
end=double_date_datetime, freq=bday_us)
dts = [x for x in dts_aux if x.to_datetime() != double_date_datetime]
shifted_datetime = dts[-shift_in_days]
return int(shifted_datetime.strftime('%Y%m%d'))
def get_binance_price_preloaded(**kwargs):
interval = kwargs['interval']
ticker = kwargs['ticker']
date_from = kwargs['date_from']
date_to = kwargs['date_to']
file_name = ticker + '_1h.pkl'
datetime_from = cu.convert_doubledate_2datetime(date_from)
datetime_to = cu.convert_doubledate_2datetime(date_to)
x = datetime_from
price_frame_list = []
while x <= datetime_to:
folder_name = dn.get_dated_directory_extension(folder_date=int(
x.strftime('%Y%m%d')),
ext='binance')
dated_file_name = folder_name + '/' + file_name
if os.path.isfile(dated_file_name):
price_frame_list.append(pd.read_pickle(dated_file_name))
x = x + dt.timedelta(days=1)
if len(price_frame_list) == 0:
return pd.DataFrame()
merged_data = pd.concat(price_frame_list)
if len(merged_data.index) == 0:
return pd.DataFrame()
merged_data.set_index('openDatetime', drop=True, inplace=True)
if interval.upper() != '1H':
data_out = pd.DataFrame()
data_out['open'] = merged_data['open'].resample('4H').first()
data_out['close'] = merged_data['close'].resample('4H').last()
data_out['high'] = merged_data['high'].resample('4H').max()
data_out['low'] = merged_data['low'].resample('4H').min()
data_out['volume'] = merged_data['volume'].resample('4H').sum()
else:
data_out = merged_data
return data_out
def get_stock_price_preloaded(**kwargs):
ticker = kwargs['ticker']
if 'data_source' in kwargs.keys():
data_source = kwargs['data_source']
else:
data_source = 'iex'
if 'stock_data_dictionary' in kwargs.keys():
data_out = kwargs['stock_data_dictionary'][ticker]
else:
if data_source == 'iex':
file_dir = dn.get_directory_name(ext='iex_stock_data')
else:
file_dir = dn.get_directory_name(ext='stock_data')
if not os.path.isfile(file_dir + '/' + ticker + '.pkl'):
ssd.save_stock_data(symbol_list=[ticker], data_source=data_source)
data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl')
report_date = exp.doubledate_shift_bus_days()
if cu.convert_doubledate_2datetime(report_date) > data_out[
'settle_datetime'].iloc[-1].to_pydatetime():
ssd.save_stock_data(symbol_list=[ticker], data_source=data_source)
data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl')
if 'settle_date' in kwargs.keys():
settle_date = kwargs['settle_date']
if isinstance(settle_date, int):
data_out = data_out[data_out['settle_datetime'] ==
cu.convert_doubledate_2datetime(settle_date)]
elif isinstance(settle_date, dt.datetime):
data_out = data_out[data_out['settle_datetime'] == settle_date]
if 'settle_date_from' in kwargs.keys():
data_out = data_out[
data_out['settle_datetime'] >= cu.convert_doubledate_2datetime(
kwargs['settle_date_from'])]
if 'settle_date_to' in kwargs.keys():
data_out = data_out[
data_out['settle_datetime'] <= cu.convert_doubledate_2datetime(
kwargs['settle_date_to'])]
return data_out
def get_signals_4date(**kwargs):
report_date = kwargs['report_date']
date_from = cu.doubledate_shift(report_date, 365)
report_datetime = cu.convert_doubledate_2datetime(report_date)
fund_price_dictionary = {
x: gsp.get_stock_price_preloaded(ticker=x,
settle_date_from=date_from,
settle_date_to=report_date)
for x in symbol_list
}
data_current_dictionary = {
x:
fund_price_dictionary[x]['settle_datetime'].iloc[-1] == report_datetime
for x in symbol_list
}
symbols2update = []
for key, value in data_current_dictionary.items():
if not value:
symbols2update.append(key)
if symbols2update:
ssd.save_stock_data(symbol_list=symbols2update)
fund_price_dictionary = {
x: gsp.get_stock_price_preloaded(ticker=x,
settle_date_from=date_from,
settle_date_to=report_date)
for x in symbol_list
}
performance_dictionary = {}
for j in range((len(symbol_list))):
price_data = fund_price_dictionary[symbol_list[j]]
if price_data['settle_datetime'].iloc[-1] != report_datetime:
return {'success': False, 'performance_dictionary': {}}
price_data.reset_index(drop=True, inplace=True)
#split_envents = price_data[price_data['split_coefficient'] != 1]
#split_index_list = split_envents.index
#for i in range(len(split_index_list)):
# price_data['close'].iloc[:split_index_list[i]] = \
# price_data['close'].iloc[:split_index_list[i]] / price_data['split_coefficient'].iloc[split_index_list[i]]
# price_data['dividend_amount'].iloc[:split_index_list[i]] = \
# price_data['dividend_amount'].iloc[:split_index_list[i]] / price_data['split_coefficient'].iloc[split_index_list[i]]
performance_dictionary[symbol_list[j]] = 100 * (
price_data['close'].iloc[-1] -
price_data['close'].iloc[0]) / price_data['close'].iloc[0]
return {'success': True, 'performance_dictionary': performance_dictionary}
def get_rolling_curve_data(**kwargs):
ticker_head = kwargs['ticker_head']
num_contracts = kwargs['num_contracts']
front_tr_dte_limit = kwargs['front_tr_dte_limit']
date_from = kwargs['date_from']
date_to = kwargs['date_to']
date_from_datetime = cu.convert_doubledate_2datetime(date_from)
date_to_datetime = cu.convert_doubledate_2datetime(date_to)
panel_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head)
panel_data = panel_data.loc[(panel_data['settle_date'] >= date_from_datetime) &
(panel_data['settle_date'] <= date_to_datetime) &
(panel_data['tr_dte'] >= front_tr_dte_limit)]
if 'month_separation' in kwargs.keys():
month_separation = kwargs['month_separation']
elif ticker_head == 'ED':
month_separation = 3
else:
month_separation = 1
if month_separation != 1:
panel_data = panel_data[panel_data['ticker_month'] % month_separation == 0]
panel_data = panel_data[np.isfinite(panel_data['close_price'])]
sorted_data = panel_data.sort(['settle_date', 'tr_dte'], ascending=[True, True])
filtered_data = sorted_data.groupby('settle_date').filter(lambda x:len(x)>=num_contracts)
filtered_data2 = filtered_data.groupby('settle_date').filter(lambda x:
all([cmi.get_month_seperation_from_cont_indx(x['cont_indx'].values[i],
x['cont_indx'].values[i+1]) ==- month_separation for i in range(num_contracts-1)]))
grouped = filtered_data2.groupby('settle_date')
rolling_data_list = []
for i in range(num_contracts):
rolling_data_list.append(grouped.nth(i))
return rolling_data_list
def get_rolling_curve_data(**kwargs):
ticker_head = kwargs['ticker_head']
num_contracts = kwargs['num_contracts']
front_tr_dte_limit = kwargs['front_tr_dte_limit']
date_from = kwargs['date_from']
date_to = kwargs['date_to']
date_from_datetime = cu.convert_doubledate_2datetime(date_from)
date_to_datetime = cu.convert_doubledate_2datetime(date_to)
panel_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head)
panel_data = panel_data.loc[(panel_data['settle_date'] >= date_from_datetime) &
(panel_data['settle_date'] <= date_to_datetime) &
(panel_data['tr_dte'] >= front_tr_dte_limit)]
if 'month_separation' in kwargs.keys():
month_separation = kwargs['month_separation']
elif ticker_head == 'ED':
month_separation = 3
else:
month_separation = 1
if month_separation != 1:
panel_data = panel_data[panel_data['ticker_month'] % month_separation == 0]
panel_data = panel_data[np.isfinite(panel_data['close_price'])]
sorted_data = panel_data.sort_values(['settle_date', 'tr_dte'], ascending=[True, True])
filtered_data = sorted_data.groupby('settle_date').filter(lambda x:len(x)>=num_contracts)
filtered_data2 = filtered_data.groupby('settle_date').filter(lambda x:
all([cmi.get_month_seperation_from_cont_indx(x['cont_indx'].values[i],
x['cont_indx'].values[i+1]) ==- month_separation for i in range(num_contracts-1)]))
grouped = filtered_data2.groupby('settle_date')
rolling_data_list = []
for i in range(num_contracts):
rolling_data_list.append(grouped.nth(i))
return rolling_data_list
def get_butterfly_panel_plot(**kwargs):
report_date = kwargs['report_date']
id = kwargs['id']
bf_output = fb.generate_futures_butterfly_sheet_4date(date_to=report_date)
butterflies = bf_output['butterflies']
contract_list = [butterflies['ticker1'][id], butterflies['ticker2'][id], butterflies['ticker3'][id]]
tr_dte_list = [butterflies['trDte1'][id], butterflies['trDte2'][id], butterflies['trDte3'][id]]
if 'aggregation_method' in kwargs.keys():
aggregation_method = kwargs['aggregation_method']
else:
aggregation_method = butterflies['agg'][id]
if 'contracts_back' in kwargs.keys():
contracts_back = kwargs['contracts_back']
else:
contracts_back = butterflies['cBack'][id]
post_report_date = exp.doubledate_shift_bus_days(double_date=report_date,shift_in_days=-20)
bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=contract_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=post_report_date,
contract_multiplier=butterflies['multiplier'][id],
use_last_as_current=True)
aligned_data = bf_signals_output['aligned_output']['aligned_data']
new_index = list(range(len(aligned_data.index)))
contract_change_indx = (aligned_data['c1']['ticker_year']-aligned_data['c1']['ticker_year'].shift(1)!=0).values
front_contract_year = aligned_data['c1']['ticker_year'] % 10
contract_change_indx[0] = False
report_datetime = cu.convert_doubledate_2datetime(report_date)
x_index = [x for x in new_index if aligned_data['settle_date'][x] == report_datetime][0]
x_tick_locations = [x for x in new_index if contract_change_indx[x]]
x_tick_locations.append(x_index)
x_tick_values = [cmi.letter_month_string[aligned_data['c1']['ticker_month'].values[x]-1]+
str(front_contract_year.values[x]) for x in new_index if contract_change_indx[x]]
x_tick_values.append('X')
plt.figure(figsize=(16, 7))
plt.plot(aligned_data['residuals'])
plt.xticks(x_tick_locations,x_tick_values)
plt.grid()
plt.title('Contracts: ' + str(contract_list) + ', weight2: ' + str(bf_signals_output['second_spread_weight_1'].round(2)))
plt.show()
return bf_signals_output
def get_butterfly_panel_plot(**kwargs):
report_date = kwargs['report_date']
id = kwargs['id']
bf_output = fb.generate_futures_butterfly_sheet_4date(date_to=report_date)
butterflies = bf_output['butterflies']
contract_list = [butterflies['ticker1'][id], butterflies['ticker2'][id], butterflies['ticker3'][id]]
tr_dte_list = [butterflies['trDte1'][id], butterflies['trDte2'][id], butterflies['trDte3'][id]]
if 'aggregation_method' in kwargs.keys():
aggregation_method = kwargs['aggregation_method']
else:
aggregation_method = butterflies['agg'][id]
if 'contracts_back' in kwargs.keys():
contracts_back = kwargs['contracts_back']
else:
contracts_back = butterflies['cBack'][id]
post_report_date = exp.doubledate_shift_bus_days(double_date=report_date,shift_in_days=-20)
bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=contract_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=post_report_date,
contract_multiplier=butterflies['multiplier'][id],
use_last_as_current=True)
aligned_data = bf_signals_output['aligned_output']['aligned_data']
new_index = list(range(len(aligned_data.index)))
contract_change_indx = (aligned_data['c1']['ticker_year']-aligned_data['c1']['ticker_year'].shift(1)!=0).values
front_contract_year = aligned_data['c1']['ticker_year'] % 10
contract_change_indx[0] = False
report_datetime = cu.convert_doubledate_2datetime(report_date)
x_index = [x for x in new_index if aligned_data['settle_date'][x] == report_datetime][0]
x_tick_locations = [x for x in new_index if contract_change_indx[x]]
x_tick_locations.append(x_index)
x_tick_values = [cmi.letter_month_string[aligned_data['c1']['ticker_month'].values[x]-1]+
str(front_contract_year.values[x]) for x in new_index if contract_change_indx[x]]
x_tick_values.append('X')
plt.figure(figsize=(16, 7))
plt.plot(aligned_data['residuals'])
plt.xticks(x_tick_locations,x_tick_values)
plt.grid()
plt.title('Contracts: ' + str(contract_list) + ', weight2: ' + str(bf_signals_output['second_spread_weight_1'].round(2)))
plt.show()
return bf_signals_output
def get_cot_data(**kwargs):
ticker_head = kwargs['ticker_head']
if os.path.isfile(presaved_cot_data_folder + '/' + ticker_head + '.pkl'):
data_out = pd.read_pickle(presaved_cot_data_folder + '/' +
ticker_head + '.pkl')
if 'date_from' in kwargs.keys():
data_out = data_out[
data_out['settle_date'] >= cu.convert_doubledate_2datetime(
kwargs['date_from'])]
if 'date_to' in kwargs.keys():
data_out = data_out[
data_out['settle_date'] <=
cu.convert_doubledate_2datetime(kwargs['date_to']) -
dt.timedelta(days=3)]
else:
data_out = pd.DataFrame()
return data_out
def get_contract_summary_stats(**kwargs):
ticker = kwargs['ticker']
date_to = kwargs['date_to']
data_out = gfp.get_futures_price_preloaded(ticker=ticker)
datetime_to = cu.convert_doubledate_2datetime(date_to)
data_out = data_out[data_out['settle_date'] <= datetime_to]
data_out['close_price_daily_diff'] = data_out['close_price'] - data_out['close_price'].shift(1)
daily_noise = np.std(data_out['close_price_daily_diff'].iloc[-60:])
average_volume = np.mean(data_out['volume'].iloc[-20:])
return {'daily_noise': daily_noise, 'average_volume':average_volume}
def get_results_4date(**kwargs):
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
output_dir = ts.create_strategy_output_dir(
strategy_class='futures_directional', report_date=date_to)
if os.path.isfile(output_dir + '/summary.pkl'):
directionals = pd.read_pickle(output_dir + '/summary.pkl')
corr_matrix = pd.read_pickle(output_dir + '/corr.pkl')
return {
'directionals': directionals,
'corr_matrix': corr_matrix,
'success': True
}
ticker_head_list = list(
set(cmi.cme_futures_tickerhead_list +
cmi.futures_butterfly_strategy_tickerhead_list))
results_output = [
get_results_4tickerhead(ticker_head=x, date_to=date_to)
for x in ticker_head_list
]
rolling_data_list = [x.pop('rolling_data') for x in results_output]
directionals = pd.DataFrame(results_output)
directionals['ticker_head'] = ticker_head_list
directionals['weekday'] = datetime_to.weekday()
directionals.to_pickle(output_dir + '/summary.pkl')
aux_frame_list = []
for i in range(len(ticker_head_list)):
rolling_data = rolling_data_list[i].iloc[-20:]
if not rolling_data.empty:
aux_frame = pd.DataFrame(index=rolling_data['settle_date'])
aux_frame[ticker_head_list[i]] = rolling_data['change_1'].values
aux_frame_list.append(aux_frame)
if len(aux_frame_list) > 1:
combined_frame = pd.concat(aux_frame_list, axis=1, join='inner')
corr_matrix = combined_frame.corr()
else:
corr_matrix = pd.DataFrame()
corr_matrix.to_pickle(output_dir + '/corr.pkl')
return {
'directionals': directionals,
'corr_matrix': corr_matrix,
'success': True
}
def get_futures_days2_expiration(expiration_input):
ticker = expiration_input['ticker']
date_to = expiration_input['date_to']
contract_specs_output = cmf.get_contract_specs(ticker)
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head']))
expiration_date = get_futures_expiration(ticker)
dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_to), end=expiration_date, freq=bday_us)
return len(dts)-1
def get_futures_days2_expiration(expiration_input):
ticker = expiration_input['ticker']
date_to = expiration_input['date_to']
contract_specs_output = cmf.get_contract_specs(ticker)
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head']))
expiration_date = get_futures_expiration(ticker)
dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_to), end=expiration_date, freq=bday_us)
return len(dts)-1
def save_price_data(**kwargs):
if 'client' in kwargs.keys():
client = kwargs['client']
else:
client = btu.get_binance_client()
interval = kwargs['interval']
ticker = kwargs['ticker']
date_from = kwargs['date_from']
date_to = kwargs['date_to']
file_name = ticker + '_' + interval + '.pkl'
datetime_from = cu.convert_doubledate_2datetime(date_from)
datetime_to = cu.convert_doubledate_2datetime(date_to)
x = datetime_from
while x <= datetime_to:
xplus = x + dt.timedelta(days=1)
folder_name = dn.get_dated_directory_extension(folder_date=int(
x.strftime('%Y%m%d')),
ext='binance')
dated_file_name = folder_name + '/' + file_name
if os.path.isfile(dated_file_name):
price_frame = pd.read_pickle(dated_file_name)
print(len(price_frame.index))
else:
price_frame = gbp.get_klines(ticker=ticker,
interval=interval,
start_str=x.strftime('%m/%d/%y'),
end_str=xplus.strftime('%m/%d/%y'),
client=client)
price_frame = price_frame[price_frame['openDate'] == x.date()]
price_frame.to_pickle(dated_file_name)
tm.sleep(0.5)
x = xplus
def get_backtesting_dates(**kwargs):
date_to = kwargs['date_to']
years_back = kwargs['years_back']
if 'day_of_week' in kwargs.keys():
day_of_week = kwargs['day_of_week']
else:
day_of_week = 2
date_from = cu.doubledate_shift(date_to, years_back*365)
trading_calendar = exp.get_calendar_4ticker_head('CL')
bday_us = CustomBusinessDay(calendar=trading_calendar)
dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from),
end=cu.convert_doubledate_2datetime(date_to), freq=bday_us)
dts = dts[dts.dayofweek==day_of_week]
return {'date_time_dates': dts,
'double_dates': [int(x.strftime('%Y%m%d')) for x in dts]}
def download_coinbase_data_4ticker(**kwargs):
date_to = kwargs["date_to"]
num_days_back = kwargs["num_days_back"]
datetime_to = cu.convert_doubledate_2datetime(date_to)
date_list = [
int((datetime_to - dt.timedelta(days=x)).strftime('%Y%m%d'))
for x in range(0, num_days_back)
]
[download_coinbase_data_4date(**kwargs, utc_date=x) for x in date_list]
def get_summary(**kwargs):
symbol1 = kwargs['symbol1']
symbol2 = kwargs['symbol2']
report_date = kwargs['report_date']
if 'get_diagnosticQ' in kwargs.keys():
get_diagnosticQ = kwargs['get_diagnosticQ']
else:
get_diagnosticQ = False
report_datetime = cu.convert_doubledate_2datetime(report_date)
data1 = gsp.get_stock_price_preloaded(ticker=symbol1, data_source='iex', settle_date_to = report_date)
data2 = gsp.get_stock_price_preloaded(ticker=symbol2, data_source='iex', settle_date_to = report_date)
merged_data = pd.merge(data1[['close','settle_datetime']], data2[['close','settle_datetime']], how='inner', on='settle_datetime')
split = int(len(merged_data) * .4)
if split<200 or report_datetime!=merged_data['settle_datetime'].iloc[-1]:
return {'price1': np.nan,'price2': np.nan, 'p_value_2': np.nan,'p_value_1': np.nan,
'beta_1': np.nan, 'beta_2': np.nan,
'corr': np.nan,
'cagr1': np.nan, 'cagr2': np.nan,
'kalpha': np.nan, 'kbeta': np.nan,
'meanSpread': np.nan, 'stdSpread': np.nan,
'zScore': np.nan}
training_data = merged_data[:split]
test_data = merged_data[split:]
cointegration_output_2 = sm.tsa.stattools.coint(training_data['close_x'], training_data['close_y'])
cointegration_output_1 = sm.tsa.stattools.coint(test_data['close_x'], test_data['close_y'])
regress_output_1 = ss.get_regression_results({'y': test_data['close_y'].values, 'x': test_data['close_x'].values})
regress_output_2 = ss.get_regression_results({'y': training_data['close_y'].values, 'x': training_data['close_x'].values})
regress_output_3 = ss.get_regression_results({'y': test_data['close_y'].diff().values, 'x': test_data['close_x'].diff().values})
merged_data.set_index('settle_datetime', drop=True, inplace=True)
backtest_output_1 = backtest(merged_data[split:], 'close_x', 'close_y')
backtest_output_2 = backtest(merged_data[:split], 'close_x', 'close_y')
if get_diagnosticQ:
return {'backtest_output': backtest_output_1, 'cagr2': backtest_output_2['cagr']}
else:
return {'price1': merged_data['close_x'].iloc[-1],'price2': merged_data['close_y'].iloc[-1],
'p_value_2': cointegration_output_2[1],'p_value_1': cointegration_output_1[1],
'beta_1': regress_output_1['beta'], 'beta_2': regress_output_2['beta'],
'corr': np.sqrt(regress_output_3['rsquared']/100),
'cagr1': backtest_output_1['cagr'], 'cagr2': backtest_output_2['cagr'],
'kalpha': backtest_output_1['kalpha'], 'kbeta': backtest_output_1['kbeta'],
'meanSpread': backtest_output_1['meanSpread'], 'stdSpread': backtest_output_1['stdSpread'],
'zScore': backtest_output_1['zScore']}
def get_bus_day_list(**kwargs):
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead =const.reference_tickerhead_4business_calendar
if 'date_from' in kwargs.keys():
datetime_from = cu.convert_doubledate_2datetime(kwargs['date_from'])
if 'datetime_from' in kwargs.keys():
datetime_from = kwargs['datetime_from']
if 'date_to' in kwargs.keys():
datetime_to = cu.convert_doubledate_2datetime(kwargs['date_to'])
if 'datetime_to' in kwargs.keys():
datetime_to = kwargs['datetime_to']
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
date_index = pd.date_range(start=datetime_from, end=datetime_to, freq=bday_us)
return [int(x.to_pydatetime().strftime('%Y%m%d')) for x in date_index]
def get_bus_day_list(**kwargs):
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead =const.reference_tickerhead_4business_calendar
if 'date_from' in kwargs.keys():
datetime_from = cu.convert_doubledate_2datetime(kwargs['date_from'])
if 'datetime_from' in kwargs.keys():
datetime_from = kwargs['datetime_from']
if 'date_to' in kwargs.keys():
datetime_to = cu.convert_doubledate_2datetime(kwargs['date_to'])
if 'datetime_to' in kwargs.keys():
datetime_to = kwargs['datetime_to']
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
date_index = pd.date_range(start=datetime_from, end=datetime_to, freq=bday_us)
return [int(x.to_datetime().strftime('%Y%m%d')) for x in date_index]
def load_trades_2strategy(**kwargs):
trade_frame = kwargs['trade_frame']
con = msu.get_my_sql_connection(**kwargs)
trade_frame['strategy_id'] = [
get_strategy_id_from_alias(alias=trade_frame['alias'][x], con=con)
for x in range(len(trade_frame.index))
]
trade_frame['strike_price'] = trade_frame['strike_price'].astype('float64')
now_time = dt.datetime.now()
now_date = now_time.date()
if 'trade_date' in kwargs.keys():
trade_date = cu.convert_doubledate_2datetime(kwargs['trade_date'])
else:
trade_date = now_date
column_str = "ticker, option_type, strike_price, strategy_id, trade_price, trade_quantity, trade_date, instrument, real_tradeQ, created_date, last_updated_date"
insert_str = ("%s, " * 11)[:-2]
final_str = "INSERT INTO trades (%s) VALUES (%s)" % (column_str,
insert_str)
column_names = trade_frame.columns.tolist()
ticker_indx = column_names.index('ticker')
option_type_indx = column_names.index('option_type')
strike_price_indx = column_names.index('strike_price')
trade_price_indx = column_names.index('trade_price')
trade_quantity_indx = column_names.index('trade_quantity')
instrument_indx = column_names.index('instrument')
real_tradeQ_indx = column_names.index('real_tradeQ')
strategy_id_indx = column_names.index('strategy_id')
tuples = [
tuple([
x[ticker_indx],
None if pd.isnull(x[option_type_indx]) else x[option_type_indx],
None if pd.isnull(x[strike_price_indx]) else x[strike_price_indx],
x[strategy_id_indx], x[trade_price_indx], x[trade_quantity_indx],
trade_date, x[instrument_indx], x[real_tradeQ_indx], now_time,
now_time
]) for x in trade_frame.values
]
msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)
if 'con' not in kwargs.keys():
con.close()
def get_days2_expiration(**kwargs):
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
expiration_datetime = get_expiration_from_db(**kwargs)
contract_specs_output = cmf.get_contract_specs(kwargs['ticker'])
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head']))
dts = pd.date_range(start=datetime_to, end=expiration_datetime, freq=bday_us)
return {'expiration_datetime': expiration_datetime,
'cal_dte': (expiration_datetime-datetime_to.date()).days,
'tr_dte': len(dts)-1}
def get_daily_price_data4ticker(**kwargs):
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
try:
data_out = pd.read_pickle(
dna.get_directory_name(ext='binance') + '/daily/' +
kwargs['ticker'] + '.pkl')
except:
return pd.DataFrame()
#data_out['pydate'] = [x.to_pydatetime().date() for x in data_out['openDatetime']]
return data_out[data_out['openDate'] <= datetime_to.date()]
def get_days2_expiration(**kwargs):
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
expiration_datetime = get_expiration_from_db(**kwargs)
contract_specs_output = cmf.get_contract_specs(kwargs['ticker'])
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head']))
dts = pd.date_range(start=datetime_to, end=expiration_datetime, freq=bday_us)
return {'expiration_datetime': expiration_datetime,
'cal_dte': (expiration_datetime-datetime_to.date()).days,
'tr_dte': len(dts)-1}
def get_backtesting_dates(**kwargs):
date_to = kwargs['date_to']
years_back = kwargs['years_back']
if 'day_of_week' in kwargs.keys():
day_of_week = kwargs['day_of_week']
else:
day_of_week = 2
date_from = cu.doubledate_shift(date_to, years_back * 365)
trading_calendar = exp.get_calendar_4ticker_head('CL')
bday_us = CustomBusinessDay(calendar=trading_calendar)
dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from),
end=cu.convert_doubledate_2datetime(date_to),
freq=bday_us)
dts = dts[dts.dayofweek == day_of_week]
return {
'date_time_dates': dts,
'double_dates': [int(x.strftime('%Y%m%d')) for x in dts]
}
def is_business_day(**kwargs):
double_date = kwargs['double_date']
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead = const.reference_tickerhead_4business_calendar
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
double_date_datetime = cu.convert_doubledate_2datetime(double_date)
dts_aux = pd.date_range(double_date_datetime, periods=1, freq=bday_us)
return double_date_datetime in dts_aux
def is_business_day(**kwargs):
double_date = kwargs['double_date']
if 'reference_tickerhead' in kwargs.keys():
reference_tickerhead = kwargs['reference_tickerhead']
else:
reference_tickerhead = const.reference_tickerhead_4business_calendar
bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead))
double_date_datetime = cu.convert_doubledate_2datetime(double_date)
dts_aux = pd.date_range(double_date_datetime, periods=1, freq=bday_us)
return double_date_datetime in dts_aux
def calc_realized_vol_4futures_ticker(**kwargs):
settle_date = kwargs.pop('settle_date')
num_obs = kwargs.pop('num_obs', 20)
futures_price_output = gfp.get_futures_price_preloaded(**kwargs)
settle_datetime = cu.convert_doubledate_2datetime(settle_date)
futures_price_selected = futures_price_output[futures_price_output['settle_date'] <= settle_datetime]
logreturns = np.log(futures_price_selected['close_price'][-(num_obs+1):]/
futures_price_selected['close_price'][-(num_obs+1):].shift(1))
return 100*np.sqrt(252*np.mean(np.square(logreturns)))
def calc_realized_vol_4futures_ticker(**kwargs):
settle_date = kwargs.pop('settle_date')
num_obs = kwargs.pop('num_obs', 20)
futures_price_output = gfp.get_futures_price_preloaded(**kwargs)
settle_datetime = cu.convert_doubledate_2datetime(settle_date)
futures_price_selected = futures_price_output[futures_price_output['settle_date'] <= settle_datetime]
logreturns = np.log(futures_price_selected['close_price'][-(num_obs+1):]/
futures_price_selected['close_price'][-(num_obs+1):].shift(1))
return 100*np.sqrt(252*np.mean(np.square(logreturns)))
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs):
ref_date = kwargs['ref_date']
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
ref_datetime = cu.convert_doubledate_2datetime(ref_date)
if aggregation_method == 12:
cal_date_list = [
ref_datetime - relativedelta(years=x)
for x in range(1, contracts_back + 1)
]
elif aggregation_method == 1:
cal_date_list = [
ref_datetime - relativedelta(months=x)
for x in range(1, contracts_back + 1)
]
bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(
const.reference_tickerhead_4business_calendar))
if 'shift_bus_days' in kwargs.keys():
shift_bus_days = kwargs['shift_bus_days']
if shift_bus_days >= 0:
bus_date_list = [
pd.date_range(x, periods=shift_bus_days + 1,
freq=bday_us)[shift_bus_days].to_pydatetime()
for x in cal_date_list
]
elif shift_bus_days < 0:
bus_date_list = [
pd.date_range(start=x - relativedelta(days=(max(
m.ceil(-shift_bus_days * 7 / 5) + 5, -shift_bus_days +
5))),
end=x,
freq=bday_us)[shift_bus_days -
1].to_pydatetime()
for x in cal_date_list
]
else:
bus_date_list = [
pd.date_range(x, periods=1, freq=bday_us)[0].to_pydatetime()
for x in cal_date_list
]
return bus_date_list
def download_coinbase_data_4date(**kwargs):
utc_date = kwargs['utc_date']
print(utc_date)
candlestick_minutes = kwargs['candlestick_minutes']
ticker = kwargs['ticker']
coinbase_data_dir = dn.get_dated_directory_extension(folder_date=utc_date,
ext='coinbase_data')
file_name = coinbase_data_dir + '/' + ticker + '_' + str(
candlestick_minutes) + '.pkl'
if os.path.isfile(file_name):
return pd.read_pickle(file_name)
if 'coin_client' in kwargs.keys():
coin_client = kwargs['coin_client']
else:
coin_client = coin_util.get_coinbase_client()
start_date = cu.convert_doubledate_2datetime(utc_date)
start_date = start_date.replace(hour=0, minute=0)
end_date = start_date + dt.timedelta(days=1)
end_date = end_date.replace(hour=0, minute=5)
date_raw = coin_client.get_product_historic_rates(
ticker,
granularity=candlestick_minutes * 60,
start=start_date,
end=end_date)
time.sleep(0.5)
frame_out = pd.DataFrame(
date_raw, columns=['time', 'low', 'high', 'open', 'close', 'volume'])
frame_out['time'] = [
dt.datetime.utcfromtimestamp(x).replace(tzinfo=pytz.utc)
for x in frame_out['time']
]
frame_out.sort_values(by='time', ascending=True, inplace=True)
frame_out.reset_index(drop=True, inplace=True)
frame_out.to_pickle(file_name)
return frame_out
def get_spread_carry_4tickerhead(**kwargs):
ticker_head = kwargs['ticker_head']
report_date = kwargs['report_date']
if 'min_tr_dte' in kwargs.keys():
min_tr_dte = kwargs['min_tr_dte']
else:
min_tr_dte = 15
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {
ticker_head:
gfp.get_futures_price_preloaded(ticker_head=ticker_head)
}
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(report_date, 5 * 365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
daily_data = gfp.get_futures_price_preloaded(
ticker_head=ticker_head,
settle_date=report_date,
futures_data_dictionary=futures_data_dictionary)
daily_data = daily_data[(daily_data['tr_dte'] >= min_tr_dte) & (
daily_data['tr_dte'] <= max_tr_dte_limits[ticker_head])]
if len(daily_data.index) > 1:
carry_signals = fs.get_futures_spread_carry_signals(
ticker_list=daily_data['ticker'].values,
tr_dte_list=daily_data['tr_dte'].values,
futures_data_dictionary=futures_data_dictionary,
datetime5_years_ago=datetime5_years_ago,
date_to=report_date)
return {'success': True, 'carry_signals': carry_signals}
else:
return {'success': False, 'carry_signals': pd.DataFrame()}
def get_altcoin_demand(**kwargs):
date_to = kwargs['date_to']
num_days_back = kwargs['num_days_back']
datetime_to = cu.convert_doubledate_2datetime(date_to)
datetime_list = [
cu.get_datetime_shift(reference_date=datetime_to, shift_in_days=x)
for x in range(num_days_back)
]
datetime_list.reverse()
date_list = [int(x.strftime('%Y%m%d')) for x in datetime_list]
demand_list = []
for i in range(len(date_list)):
ticker_frame_out = gd.get_ticker_frame(date=date_list[i])
btc_frame = ticker_frame_out.loc[[
x[-3:] == 'BTC' for x in ticker_frame_out['symbol']
]]
btc_frame['quoteVolume'] = pd.to_numeric(btc_frame['quoteVolume'])
btc_frame['priceChangePercent'] = pd.to_numeric(
btc_frame['priceChangePercent'])
btc_frame.sort_values(['quoteVolume'], inplace=True, ascending=False)
btc_frame = btc_frame.iloc[:50]
btc_frame['averageVolume'] = [
get_average_volume(ticker=x, date_to=date_list[i])
for x in btc_frame['symbol']
]
btc_frame.sort_values(['averageVolume'], inplace=True, ascending=False)
btc_frame = btc_frame.iloc[:10]
demand_list.append(np.mean(btc_frame['priceChangePercent']))
data_out = pd.DataFrame()
data_out['date'] = date_list
data_out['datetime'] = datetime_list
data_out['demand'] = demand_list
data_out['cumulative'] = data_out['demand'].cumsum()
data_out['pydate'] = [x.date() for x in data_out['datetime']]
return data_out
def get_tickers_from_agg_method_and_contracts_back(**kwargs):
ticker = kwargs['ticker']
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
contact_specs_out = cmi.get_contract_specs(ticker)
ref_date = 10000*contact_specs_out['ticker_year']+100*contact_specs_out['ticker_month_num']+1
ref_datetime = cu.convert_doubledate_2datetime(ref_date)
if aggregation_method == 12:
cal_date_list = [ref_datetime - relativedelta(years=x) for x in range(1, contracts_back+1)]
elif aggregation_method == 1:
cal_date_list = [ref_datetime - relativedelta(months=x) for x in range(1, contracts_back+1)]
ticker_list = [contact_specs_out['ticker_head'] + cmi.full_letter_month_list[x.month-1] +
str(x.year) for x in cal_date_list]
return ticker_list
def get_intraday_breakout_chart(**kwargs):
signal_out = ifs.get_intraday_trend_signals(ticker=kwargs['ticker'],date_to=kwargs['trade_date'])
intraday_data = signal_out['intraday_data']
trade_datetime = cu.convert_doubledate_2datetime(kwargs['trade_date'])
intraday_data = intraday_data[intraday_data['time_stamp']>=trade_datetime]
plt.figure(figsize=(16, 7))
plt.plot(intraday_data['time_stamp'],intraday_data['mid_p'],color='k')
plt.plot(intraday_data['time_stamp'],intraday_data['ewma25'],color='b')
plt.plot(intraday_data['time_stamp'],intraday_data['ewma100'],color='g')
plt.axvline(dt.datetime.combine(trade_datetime,dt.time(8,30,0,0)),color='r')
plt.axvline(dt.datetime.combine(trade_datetime,dt.time(9,0,0,0)),color='r')
#plt.axvline([x for x in intraday_data['time_stamp'] if 100*x.hour+x.minute == 830][0],color='r')
#plt.axvline([x for x in intraday_data.index if 100*x.hour+x.minute == 900][0],color='r')
plt.grid()
plt.show()
def load_trades_2strategy(**kwargs):
trade_frame = kwargs['trade_frame']
con = msu.get_my_sql_connection(**kwargs)
trade_frame['strategy_id'] = [get_strategy_id_from_alias(alias=trade_frame['alias'][x],con=con) for x in range(len(trade_frame.index))]
trade_frame['strike_price'] = trade_frame['strike_price'].astype('float64')
now_time = dt.datetime.now()
now_date = now_time.date()
if 'trade_date' in kwargs.keys():
trade_date = cu.convert_doubledate_2datetime(kwargs['trade_date'])
else:
trade_date = now_date
column_str = "ticker, option_type, strike_price, strategy_id, trade_price, trade_quantity, trade_date, instrument, real_tradeQ, created_date, last_updated_date"
insert_str = ("%s, " * 11)[:-2]
final_str = "INSERT INTO trades (%s) VALUES (%s)" % (column_str, insert_str)
column_names = trade_frame.columns.tolist()
ticker_indx = column_names.index('ticker')
option_type_indx = column_names.index('option_type')
strike_price_indx = column_names.index('strike_price')
trade_price_indx = column_names.index('trade_price')
trade_quantity_indx = column_names.index('trade_quantity')
instrument_indx = column_names.index('instrument')
real_tradeQ_indx = column_names.index('real_tradeQ')
strategy_id_indx = column_names.index('strategy_id')
tuples = [tuple([x[ticker_indx],x[option_type_indx],
None if np.isnan(x[strike_price_indx]) else x[strike_price_indx],
x[strategy_id_indx],x[trade_price_indx], x[trade_quantity_indx],
trade_date,x[instrument_indx], x[real_tradeQ_indx],now_time,now_time]) for x in trade_frame.values]
msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)
if 'con' not in kwargs.keys():
con.close()
def generate_db_strategy_from_alias(**kwargs):
alias = kwargs['alias']
description_string = kwargs['description_string']
con = msu.get_my_sql_connection(**kwargs)
cur = con.cursor()
for i in range(1, 50):
if i > 1:
alias_modified = alias + '_' + str(i)
else:
alias_modified = alias
strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con)
if not strategy_id:
break
now_date = dt.datetime.now().date()
column_str = "alias, open_date, close_date, created_date, last_updated_date, description_string"
insert_str = ("%s, " * 6)[:-2]
tuple_to_load = (alias_modified, now_date,
cu.convert_doubledate_2datetime(30000101), now_date,
now_date, description_string)
final_str = "INSERT INTO strategy (%s) VALUES (%s)" % (column_str,
insert_str)
cur.execute(final_str, tuple_to_load)
con.commit()
strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con)
if 'con' not in kwargs.keys():
con.close()
return {'alias': alias_modified, 'strategy_id': strategy_id}
def get_net_position_4strategy_alias(**kwargs):
alias = kwargs['alias']
if 'as_of_date' in kwargs.keys():
as_of_date = kwargs['as_of_date']
else:
as_of_date = exp.doubledate_shift_bus_days()
con = msu.get_my_sql_connection(**kwargs)
trades_frame = get_trades_4strategy_alias(alias=alias, con=con)
as_of_datetime = cu.convert_doubledate_2datetime(as_of_date)
trades_frame = trades_frame[trades_frame['trade_date'] <= as_of_datetime]
trades_frame['full_ticker'] = [
trades_frame['ticker'].iloc[x] if trades_frame['instrument'].iloc[x]
in ['F', 'S'] else trades_frame['ticker'].iloc[x] + '_' +
trades_frame['option_type'].iloc[x] +
str(trades_frame['strike_price'].iloc[x])
for x in range(len(trades_frame.index))
]
grouped = trades_frame.groupby('full_ticker')
net_position = pd.DataFrame()
net_position['ticker'] = (grouped['ticker'].first()).values
net_position['option_type'] = (grouped['option_type'].first()).values
net_position['strike_price'] = (grouped['strike_price'].first()).values
net_position['instrument'] = (grouped['instrument'].first()).values
net_position['qty'] = (grouped['trade_quantity'].sum()).values
net_position['qty'] = net_position['qty'].round(2)
if 'con' not in kwargs.keys():
con.close()
return net_position[net_position['qty'] != 0]
def get_spread_carry_4tickerhead(**kwargs):
ticker_head = kwargs['ticker_head']
report_date = kwargs['report_date']
if 'min_tr_dte' in kwargs.keys():
min_tr_dte = kwargs['min_tr_dte']
else:
min_tr_dte = 10
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {ticker_head: gfp.get_futures_price_preloaded(ticker_head=ticker_head)}
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(report_date,5*365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
daily_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head,
settle_date=report_date,
futures_data_dictionary=futures_data_dictionary)
daily_data = daily_data[(daily_data['tr_dte'] >= min_tr_dte) & (daily_data['tr_dte'] <= max_tr_dte_limits[ticker_head])]
if len(daily_data.index) > 1:
carry_signals = fs.get_futures_spread_carry_signals(ticker_list=daily_data['ticker'].values,
tr_dte_list=daily_data['tr_dte'].values,
futures_data_dictionary=futures_data_dictionary,
datetime5_years_ago=datetime5_years_ago,
date_to=report_date)
return {'success': True, 'carry_signals': carry_signals}
else:
return {'success': False, 'carry_signals': pd.DataFrame()}
def generate_db_strategy_from_alias(**kwargs):
alias = kwargs['alias']
description_string = kwargs['description_string']
con = msu.get_my_sql_connection(**kwargs)
cur = con.cursor()
for i in range(1, 10):
if i > 1:
alias_modified = alias + '_' + str(i)
else:
alias_modified = alias
strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con)
if not strategy_id:
break
now_date = dt.datetime.now().date()
column_str = "alias, open_date, close_date, created_date, last_updated_date, description_string"
insert_str = ("%s, " * 6)[:-2]
tuple_to_load = (alias_modified, now_date, cu.convert_doubledate_2datetime(30000101),
now_date, now_date, description_string)
final_str = "INSERT INTO strategy (%s) VALUES (%s)" % (column_str, insert_str)
cur.execute(final_str,tuple_to_load)
con.commit()
strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con)
if 'con' not in kwargs.keys():
con.close()
return strategy_id
def get_net_position_4strategy_alias(**kwargs):
alias = kwargs['alias']
if 'as_of_date' in kwargs.keys():
as_of_date = kwargs['as_of_date']
else:
as_of_date = exp.doubledate_shift_bus_days()
con = msu.get_my_sql_connection(**kwargs)
trades_frame = get_trades_4strategy_alias(alias=alias,con=con)
as_of_datetime = cu.convert_doubledate_2datetime(as_of_date)
trades_frame = trades_frame[trades_frame['trade_date'] <= as_of_datetime]
trades_frame['full_ticker'] = [trades_frame['ticker'].iloc[x] if trades_frame['instrument'].iloc[x] == 'F' else
trades_frame['ticker'].iloc[x] + '_' + trades_frame['option_type'].iloc[x] + str(trades_frame['strike_price'].iloc[x])
for x in range(len(trades_frame.index))]
grouped = trades_frame.groupby('full_ticker')
net_position = pd.DataFrame()
net_position['ticker'] = (grouped['ticker'].first()).values
net_position['option_type'] = (grouped['option_type'].first()).values
net_position['strike_price'] = (grouped['strike_price'].first()).values
net_position['instrument'] = (grouped['instrument'].first()).values
net_position['qty'] = (grouped['trade_quantity'].sum()).values
net_position['qty'] = net_position['qty'].round(2)
if 'con' not in kwargs.keys():
con.close()
return net_position[net_position['qty'] != 0]
def generate_futures_butterfly_sheet_4date(**kwargs):
date_to = kwargs['date_to']
output_dir = ts.create_strategy_output_dir(strategy_class='futures_butterfly', report_date=date_to)
if os.path.isfile(output_dir + '/summary.pkl'):
butterflies = pd.read_pickle(output_dir + '/summary.pkl')
return {'butterflies': butterflies,'success': True}
if 'volume_filter' not in kwargs.keys():
kwargs['volume_filter'] = 100
butterflies = get_futures_butterflies_4date(**kwargs)
butterflies = butterflies[butterflies['trDte1'] >= 35]
butterflies.reset_index(drop=True,inplace=True)
num_butterflies = len(butterflies)
q_list = [None]*num_butterflies
qf_list = [None]*num_butterflies
zscore1_list = [None]*num_butterflies
zscore2_list = [None]*num_butterflies
zscore3_list = [None]*num_butterflies
zscore4_list = [None]*num_butterflies
zscore5_list = [None]*num_butterflies
zscore6_list = [None]*num_butterflies
zscore7_list = [None]*num_butterflies
rsquared1_list = [None]*num_butterflies
rsquared2_list = [None]*num_butterflies
regime_change_list = [None]*num_butterflies
contract_seasonality_list = [None]*num_butterflies
yield1_list = [None]*num_butterflies
yield2_list = [None]*num_butterflies
bf_price_list = [None]*num_butterflies
bf_price_sell_limit_list = [None]*num_butterflies
bf_price_buy_limit_list = [None]*num_butterflies
noise_ratio_list = [None]*num_butterflies
alpha1_list = [None]*num_butterflies
alpha2_list = [None]*num_butterflies
residual_std1_list = [None]*num_butterflies
residual_std2_list = [None]*num_butterflies
second_spread_weight_1_list = [None]*num_butterflies
second_spread_weight_2_list = [None]*num_butterflies
weight1_list = [None]*num_butterflies
weight2_list = [None]*num_butterflies
weight3_list = [None]*num_butterflies
downside_list = [None]*num_butterflies
upside_list = [None]*num_butterflies
recent_5day_pnl_list = [None]*num_butterflies
recent_vol_ratio_list = [None]*num_butterflies
theo_pnl_list = [None]*num_butterflies
theo_pnl5_list = [None]*num_butterflies
theo_pnl10_list = [None]*num_butterflies
theo_pnl15_list = [None]*num_butterflies
theo_pnl20_list = [None]*num_butterflies
theo_pnl25_list = [None]*num_butterflies
ratio_target5_list = [None]*num_butterflies
ratio_target10_list = [None]*num_butterflies
ratio_target15_list = [None]*num_butterflies
ratio_target20_list = [None]*num_butterflies
ratio_target25_list = [None]*num_butterflies
price_1_list = [None]*num_butterflies
price_2_list = [None]*num_butterflies
price_3_list = [None]*num_butterflies
mean_reversion_rsquared_list = [None]*num_butterflies
mean_reversion_signif_list = [None]*num_butterflies
futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in cmi.futures_butterfly_strategy_tickerhead_list}
date5_years_ago = cu.doubledate_shift(date_to,5*365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
for i in range(num_butterflies):
bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=[butterflies['ticker1'][i], butterflies['ticker2'][i], butterflies['ticker3'][i]],
tr_dte_list=[butterflies['trDte1'][i], butterflies['trDte2'][i], butterflies['trDte3'][i]],
aggregation_method=butterflies['agg'][i],
contracts_back=butterflies['cBack'][i],
date_to=date_to,
futures_data_dictionary=futures_data_dictionary,
contract_multiplier=butterflies['multiplier'][i],
datetime5_years_ago=datetime5_years_ago)
q_list[i] = bf_signals_output['q']
qf_list[i] = bf_signals_output['qf']
zscore1_list[i] = bf_signals_output['zscore1']
zscore2_list[i] = bf_signals_output['zscore2']
zscore3_list[i] = bf_signals_output['zscore3']
zscore4_list[i] = bf_signals_output['zscore4']
zscore5_list[i] = bf_signals_output['zscore5']
zscore6_list[i] = bf_signals_output['zscore6']
zscore7_list[i] = bf_signals_output['zscore7']
rsquared1_list[i] = bf_signals_output['rsquared1']
rsquared2_list[i] = bf_signals_output['rsquared2']
regime_change_list[i] = bf_signals_output['regime_change_ind']
contract_seasonality_list[i] = bf_signals_output['contract_seasonality_ind']
yield1_list[i] = bf_signals_output['yield1_current']
yield2_list[i] = bf_signals_output['yield2_current']
bf_price_list[i] = bf_signals_output['bf_price']
bf_price_sell_limit_list[i] = bf_signals_output['short_price_limit']
bf_price_buy_limit_list[i] = bf_signals_output['long_price_limit']
noise_ratio_list[i] = bf_signals_output['noise_ratio']
alpha1_list[i] = bf_signals_output['alpha1']
alpha2_list[i] = bf_signals_output['alpha2']
residual_std1_list = bf_signals_output['residual_std1']
residual_std2_list = bf_signals_output['residual_std2']
second_spread_weight_1_list[i] = bf_signals_output['second_spread_weight_1']
second_spread_weight_2_list[i] = bf_signals_output['second_spread_weight_2']
weight1_list[i] = bf_signals_output['weight1']
weight2_list[i] = bf_signals_output['weight2']
weight3_list[i] = bf_signals_output['weight3']
downside_list[i] = bf_signals_output['downside']
upside_list[i] = bf_signals_output['upside']
recent_5day_pnl_list[i] = bf_signals_output['recent_5day_pnl']
recent_vol_ratio_list[i] = bf_signals_output['recent_vol_ratio']
theo_pnl_list[i] = bf_signals_output['theo_pnl']
theo_pnl5_list[i] = bf_signals_output['theo_pnl_list'][0]
theo_pnl10_list[i] = bf_signals_output['theo_pnl_list'][1]
theo_pnl15_list[i] = bf_signals_output['theo_pnl_list'][2]
theo_pnl20_list[i] = bf_signals_output['theo_pnl_list'][3]
theo_pnl25_list[i] = bf_signals_output['theo_pnl_list'][4]
ratio_target5_list[i] = bf_signals_output['ratio_target_list'][0]
ratio_target10_list[i] = bf_signals_output['ratio_target_list'][1]
ratio_target15_list[i] = bf_signals_output['ratio_target_list'][2]
ratio_target20_list[i] = bf_signals_output['ratio_target_list'][3]
ratio_target25_list[i] = bf_signals_output['ratio_target_list'][4]
price_1_list[i] = bf_signals_output['price_1']
price_2_list[i] = bf_signals_output['price_2']
price_3_list[i] = bf_signals_output['price_3']
mean_reversion_rsquared_list[i] = bf_signals_output['mean_reversion_rsquared']
mean_reversion_signif_list[i] = bf_signals_output['mean_reversion_signif']
butterflies['Q'] = q_list
butterflies['QF'] = qf_list
butterflies['z1'] = zscore1_list
butterflies['z2'] = zscore2_list
butterflies['z3'] = zscore3_list
butterflies['z4'] = zscore4_list
butterflies['z5'] = zscore5_list
butterflies['z6'] = zscore6_list
butterflies['z7'] = zscore7_list
butterflies['r1'] = rsquared1_list
butterflies['r2'] = rsquared2_list
butterflies['RC'] = regime_change_list
butterflies['seasonality'] = contract_seasonality_list
butterflies['yield1'] = yield1_list
butterflies['yield2'] = yield2_list
butterflies['bf_price'] = bf_price_list
butterflies['bf_sell_limit'] = bf_price_sell_limit_list
butterflies['bf_buy_limit'] = bf_price_buy_limit_list
butterflies['noise_ratio'] = noise_ratio_list
butterflies['alpha1'] = alpha1_list
butterflies['alpha2'] = alpha2_list
butterflies['residual_std1'] = residual_std1_list
butterflies['residual_std2'] = residual_std2_list
butterflies['second_spread_weight_1'] = second_spread_weight_1_list
butterflies['second_spread_weight_2'] = second_spread_weight_2_list
butterflies['weight1'] = weight1_list
butterflies['weight2'] = weight2_list
butterflies['weight3'] = weight3_list
butterflies['downside'] = downside_list
butterflies['upside'] = upside_list
butterflies['recent_5day_pnl'] = recent_5day_pnl_list
butterflies['recent_vol_ratio'] = recent_vol_ratio_list
butterflies['theo_pnl'] = theo_pnl_list
butterflies['theo_pnl5'] = theo_pnl5_list
butterflies['theo_pnl10'] = theo_pnl10_list
butterflies['theo_pnl15'] = theo_pnl15_list
butterflies['theo_pnl20'] = theo_pnl20_list
butterflies['theo_pnl25'] = theo_pnl25_list
butterflies['ratio_target5'] = ratio_target5_list
butterflies['ratio_target10'] = ratio_target10_list
butterflies['ratio_target15'] = ratio_target15_list
butterflies['ratio_target20'] = ratio_target20_list
butterflies['ratio_target25'] = ratio_target25_list
butterflies['price1'] = price_1_list
butterflies['price2'] = price_2_list
butterflies['price3'] = price_3_list
butterflies['mean_reversion_rsquared'] = mean_reversion_rsquared_list
butterflies['mean_reversion_signif'] = mean_reversion_signif_list
butterflies['z1'] = butterflies['z1'].round(2)
butterflies['z2'] = butterflies['z2'].round(2)
butterflies['z3'] = butterflies['z3'].round(2)
butterflies['z4'] = butterflies['z4'].round(2)
butterflies['z5'] = butterflies['z5'].round(2)
butterflies['z6'] = butterflies['z6'].round(2)
butterflies['z7'] = butterflies['z7'].round(2)
butterflies['r1'] = butterflies['r1'].round(2)
butterflies['r2'] = butterflies['r2'].round(2)
butterflies['RC'] = butterflies['RC'].round(2)
butterflies['seasonality'] = butterflies['seasonality'].round(2)
butterflies['second_spread_weight_1'] = butterflies['second_spread_weight_1'].round(2)
butterflies['second_spread_weight_2'] = butterflies['second_spread_weight_1'].round(2)
butterflies['yield1'] = butterflies['yield1'].round(3)
butterflies['yield2'] = butterflies['yield2'].round(3)
butterflies['noise_ratio'] = butterflies['noise_ratio'].round(3)
butterflies['alpha1'] = butterflies['alpha1'].round(3)
butterflies['alpha2'] = butterflies['alpha2'].round(3)
butterflies['residual_std1'] = butterflies['residual_std1'].round(3)
butterflies['residual_std2'] = butterflies['residual_std2'].round(3)
butterflies['downside'] = butterflies['downside'].round(3)
butterflies['upside'] = butterflies['upside'].round(3)
butterflies['recent_5day_pnl'] = butterflies['recent_5day_pnl'].round(3)
butterflies['recent_vol_ratio'] = butterflies['recent_vol_ratio'].round(2)
butterflies['theo_pnl'] = butterflies['theo_pnl'].round(3)
butterflies['price1'] = butterflies['price1'].round(4)
butterflies['price2'] = butterflies['price2'].round(4)
butterflies['price3'] = butterflies['price3'].round(4)
butterflies['mean_reversion_rsquared'] = butterflies['mean_reversion_rsquared'].round(2)
butterflies.to_pickle(output_dir + '/summary.pkl')
return {'butterflies': butterflies,'success': True}
def get_aligned_option_indicators_legacy(**kwargs):
ticker_list = kwargs['ticker_list']
tr_dte_list = kwargs['tr_dte_list']
settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date'])
if 'num_cal_days_back' in kwargs.keys():
num_cal_days_back = kwargs['num_cal_days_back']
else:
num_cal_days_back = 20*365
settle_datetime_from = settle_datetime-dt.timedelta(num_cal_days_back)
contract_specs_output_list = [cmi.get_contract_specs(x) for x in ticker_list]
ticker_head_list = [x['ticker_head'] for x in contract_specs_output_list]
cont_indx_list = [x['ticker_year']*100+x['ticker_month_num'] for x in contract_specs_output_list]
month_seperation_list = [cmi.get_month_seperation_from_cont_indx(x,cont_indx_list[0]) for x in cont_indx_list]
aggregation_method = max([ocu.get_aggregation_method_contracts_back({'ticker_class': x['ticker_class'],
'ticker_head': x['ticker_head']})['aggregation_method'] for x in contract_specs_output_list])
if (min(tr_dte_list) >= 80) and (aggregation_method == 1):
aggregation_method = 3
tr_days_half_band_width_selected = ocu.tr_days_half_band_with[aggregation_method]
data_frame_list = []
for x in range(len(ticker_list)):
if ticker_head_list[x] in ['ED', 'E0', 'E2', 'E3', 'E4', 'E5']:
model = 'OU'
else:
model = 'BS'
tr_dte_upper_band = tr_dte_list[x]+tr_days_half_band_width_selected
tr_dte_lower_band = tr_dte_list[x]-tr_days_half_band_width_selected
ref_tr_dte_list = [y for y in cmi.aligned_data_tr_dte_list if y <= tr_dte_upper_band and y>=tr_dte_lower_band]
if len(ref_tr_dte_list) == 0:
return {'hist': [], 'current': [], 'success': False}
if aggregation_method == 12:
aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]],
tr_dte_center=y,
contract_month_letter=contract_specs_output_list[x]['ticker_month_str'],
model=model) for y in ref_tr_dte_list]
else:
aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]],
tr_dte_center=y,
model=model) for y in ref_tr_dte_list]
aligned_data = [y[(y['trDTE'] >= tr_dte_lower_band)&(y['trDTE'] <= tr_dte_upper_band)] for y in aligned_data]
aligned_data = pd.concat(aligned_data)
aligned_data.drop('theta', axis=1, inplace=True)
aligned_data['settle_date'] = pd.to_datetime(aligned_data['settleDates'].astype('str'), format='%Y%m%d')
aligned_data = aligned_data[(aligned_data['settle_date'] <= settle_datetime)&(aligned_data['settle_date'] >= settle_datetime_from)]
aligned_data.rename(columns={'TickerYear': 'ticker_year',
'TickerMonth': 'ticker_month',
'trDTE': 'tr_dte',
'calDTE': 'cal_dte',
'impVol': 'imp_vol',
'close2CloseVol20': 'close2close_vol20',
'dollarTheta': 'theta'}, inplace=True)
aligned_data.sort(['settle_date', 'ticker_year', 'ticker_month'], ascending=[True,True,True],inplace=True)
aligned_data.drop_duplicates(['settle_date','ticker_year','ticker_month'],inplace=True)
aligned_data = aligned_data[['settle_date','ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'theta', 'close2close_vol20', 'profit5']]
aligned_data['cont_indx'] = 100*aligned_data['ticker_year']+aligned_data['ticker_month']
aligned_data['cont_indx_adj'] = [cmi.get_cont_indx_from_month_seperation(y,-month_seperation_list[x]) for y in aligned_data['cont_indx']]
data_frame_list.append(aligned_data)
for x in range(len(ticker_list)):
data_frame_list[x].set_index(['settle_date','cont_indx_adj'], inplace=True,drop=False)
merged_dataframe = pd.concat(data_frame_list, axis=1, join='inner',keys=['c'+ str(x+1) for x in range(len(ticker_list))])
merged_dataframe['abs_tr_dte_diff'] = abs(merged_dataframe['c1']['tr_dte']-tr_dte_list[0])
merged_dataframe['settle_date'] = merged_dataframe['c1']['settle_date']
merged_dataframe.sort(['settle_date', 'abs_tr_dte_diff'], ascending=[True,True], inplace=False)
merged_dataframe.drop_duplicates('settle_date', inplace=True, take_last=False)
merged_dataframe.index = merged_dataframe.index.droplevel(1)
tr_dte_list = []
cal_dte_list = []
imp_vol_list = []
theta_list = []
close2close_vol20_list = []
for x in range(len(ticker_list)):
selected_data = merged_dataframe['c' + str(x+1)]
if settle_datetime in selected_data.index:
selected_data = selected_data.loc[settle_datetime]
else:
return {'hist': [], 'current': [], 'success': False}
if selected_data['cont_indx'] != cont_indx_list[x]:
return {'hist': [], 'current': [], 'success': False}
tr_dte_list.append(selected_data['tr_dte'])
cal_dte_list.append(selected_data['cal_dte'])
imp_vol_list.append(selected_data['imp_vol'])
theta_list.append(selected_data['theta'])
close2close_vol20_list.append(selected_data['close2close_vol20'])
current_data = pd.DataFrame.from_items([('ticker',ticker_list),
('tr_dte', tr_dte_list),
('cal_dte', cal_dte_list),
('imp_vol', imp_vol_list),
('theta', theta_list),
('close2close_vol20', close2close_vol20_list)])
current_data['settle_date'] = settle_datetime
current_data.set_index('ticker', drop=True, inplace=True)
return {'hist': merged_dataframe, 'current': current_data, 'success': True}
def get_strategy_pnl_4day(**kwargs):
alias = kwargs['alias']
pnl_date = kwargs['pnl_date']
#print(pnl_date)
pnl_datetime = cu.convert_doubledate_2datetime(pnl_date)
con = msu.get_my_sql_connection(**kwargs)
if 'trades_frame' in kwargs.keys():
trades_frame = kwargs['trades_frame']
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in trades_frame['ticker']]
else:
trades_frame = ts.get_trades_4strategy_alias(alias=alias,con=con)
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in trades_frame['ticker']]
trades_frame['contract_multiplier'] = [cmi.contract_multiplier[x] for x in ticker_head_list]
trades_frame['t_cost'] = [cmi.t_cost[x] for x in ticker_head_list]
trades_frame['ticker_head'] = ticker_head_list
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
unique_ticker_head_list = list(set(ticker_head_list))
futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in unique_ticker_head_list}
pnl_date_1 = exp.doubledate_shift_bus_days(double_date=pnl_date)
underlying_frame = trades_frame[trades_frame['instrument'] == 'F']
option_frame = trades_frame[trades_frame['instrument'] == 'O']
futures_price_out_1 = [gfp.get_futures_price_preloaded(ticker=x,
futures_data_dictionary=futures_data_dictionary,
settle_date=pnl_date_1) for x in underlying_frame['ticker']]
futures_price_out = [gfp.get_futures_price_preloaded(ticker=x,
futures_data_dictionary=futures_data_dictionary,
settle_date=pnl_date) for x in underlying_frame['ticker']]
underlying_frame['price_1'] = [np.NaN if x.empty else x['close_price'].values[0] for x in futures_price_out_1]
underlying_frame['price'] = [np.NaN if x.empty else x['close_price'].values[0] for x in futures_price_out]
#underlying_frame['price_1'] = [gfp.get_futures_price_preloaded(ticker=x,
# futures_data_dictionary=futures_data_dictionary,
# settle_date=pnl_date_1)['close_price'].values[0] for x in underlying_frame['ticker']]
#underlying_frame['price'] = [gfp.get_futures_price_preloaded(ticker=x,
# futures_data_dictionary=futures_data_dictionary,
# settle_date=pnl_date)['close_price'].values[0] for x in underlying_frame['ticker']]
option_frame['price_1'] = [gop.get_options_price_from_db(ticker=option_frame['ticker'].iloc[x],
strike=option_frame['strike_price'].iloc[x],
option_type=option_frame['option_type'].iloc[x],
con=con,settle_date=pnl_date_1,
return_nan_if_emptyQ = True)['close_price'].values[0] for x in range(len(option_frame.index))]
option_frame['price'] = [gop.get_options_price_from_db(ticker=option_frame['ticker'].iloc[x],
strike=option_frame['strike_price'].iloc[x],
option_type=option_frame['option_type'].iloc[x],
con=con,settle_date=pnl_date,
return_nan_if_emptyQ = True)['close_price'].values[0] for x in range(len(option_frame.index))]
trades_frame = pd.concat([option_frame, underlying_frame])
position_frame = trades_frame[trades_frame['trade_date'] < pnl_datetime]
intraday_frame = trades_frame[trades_frame['trade_date'] == pnl_datetime]
position_pnl_per_ticker = pd.DataFrame(columns=['ticker','pnl_position'])
intraday_pnl_per_ticker = pd.DataFrame(columns=['ticker','pnl_intraday'])
position_pnl_per_tickerhead = pd.DataFrame(columns=['ticker_head','pnl_position'])
intraday_pnl_per_tickerhead = pd.DataFrame(columns=['ticker_head','pnl_intraday'])
if len(position_frame) == 0:
position_pnl = 0
else:
position_frame['pnl'] = position_frame['contract_multiplier']*\
position_frame['trade_quantity']*\
(position_frame['price']-position_frame['price_1'])
position_pnl = position_frame['pnl'].sum()
position_grouped_per_ticker = position_frame.groupby('ticker')
position_grouped_per_tickerhead = position_frame.groupby('ticker_head')
position_pnl_per_ticker['pnl_position'] = (position_grouped_per_ticker['pnl'].sum()).values
position_pnl_per_ticker['ticker'] = (position_grouped_per_ticker['ticker'].first()).values
position_pnl_per_tickerhead['pnl_position'] = (position_grouped_per_tickerhead['pnl'].sum()).values
position_pnl_per_tickerhead['ticker_head'] = (position_grouped_per_tickerhead['ticker_head'].first()).values
if len(intraday_frame) == 0:
intraday_pnl = 0
t_cost = 0
else:
intraday_frame['pnl'] = intraday_frame['contract_multiplier']*\
intraday_frame['trade_quantity']*\
(intraday_frame['price']-intraday_frame['trade_price'])
intraday_frame['pnl_wtcost'] = intraday_frame['pnl']-abs(intraday_frame['trade_quantity']*intraday_frame['t_cost'])
intraday_pnl = intraday_frame['pnl'].sum()
t_cost = (abs(intraday_frame['trade_quantity']*intraday_frame['t_cost'])).sum()
intraday_grouped_per_ticker = intraday_frame.groupby('ticker')
intraday_grouped_per_tickerhead = intraday_frame.groupby('ticker_head')
intraday_pnl_per_ticker['pnl_intraday'] = (intraday_grouped_per_ticker['pnl_wtcost'].sum()).values
intraday_pnl_per_ticker['ticker'] = (intraday_grouped_per_ticker['ticker'].first()).values
intraday_pnl_per_tickerhead['pnl_intraday'] = (intraday_grouped_per_tickerhead['pnl_wtcost'].sum()).values
intraday_pnl_per_tickerhead['ticker_head'] = (intraday_grouped_per_tickerhead['ticker_head'].first()).values
pnl_per_ticker = pd.merge(position_pnl_per_ticker,intraday_pnl_per_ticker,how='outer',on='ticker')
intraday_zero_indx = [x not in intraday_pnl_per_ticker['ticker'].values for x in pnl_per_ticker['ticker']]
position_zero_indx = [x not in position_pnl_per_ticker['ticker'].values for x in pnl_per_ticker['ticker']]
pnl_per_ticker['pnl_position'][position_zero_indx] = 0
pnl_per_ticker['pnl_intraday'][intraday_zero_indx] = 0
pnl_per_ticker['pnl_total'] = pnl_per_ticker['pnl_position']+pnl_per_ticker['pnl_intraday']
pnl_per_ticker.set_index('ticker', drop=True, inplace=True)
pnl_per_tickerhead = pd.merge(position_pnl_per_tickerhead,intraday_pnl_per_tickerhead,how='outer',on='ticker_head')
intraday_zero_indx = [x not in intraday_pnl_per_tickerhead['ticker_head'].values for x in pnl_per_tickerhead['ticker_head']]
position_zero_indx = [x not in position_pnl_per_tickerhead['ticker_head'].values for x in pnl_per_tickerhead['ticker_head']]
pnl_per_tickerhead['pnl_position'][position_zero_indx] = 0
pnl_per_tickerhead['pnl_intraday'][intraday_zero_indx] = 0
pnl_per_tickerhead['pnl_total'] = pnl_per_tickerhead['pnl_position']+pnl_per_tickerhead['pnl_intraday']
pnl_per_tickerhead.set_index('ticker_head', drop=True, inplace=True)
if 'con' not in kwargs.keys():
con.close()
return {'total_pnl': int(position_pnl+intraday_pnl - t_cost),
'position_pnl': int(position_pnl),
'intraday_pnl': int(intraday_pnl),
't_cost': int(t_cost),
'pnl_per_ticker': pnl_per_ticker,
'pnl_per_tickerhead':pnl_per_tickerhead}
def update_options_price_database_from_cme_files_4ticker(**kwargs):
ticker = kwargs['ticker']
contract_specs_output = cmi.get_contract_specs(ticker)
ticker_head = contract_specs_output['ticker_head']
ticker_month_num = contract_specs_output['ticker_month_num']
ticker_year = contract_specs_output['ticker_year']
if 'settle_date' in kwargs.keys():
settle_date = kwargs['settle_date']
kwargs['report_date'] = settle_date
else:
settle_date = int(time.strftime('%Y%m%d'))
kwargs['settle_date'] = settle_date
kwargs['report_date'] = settle_date
if not exp.is_business_day(double_date=settle_date, reference_tickerhead=ticker_head):
return
if 'expiration_date' in kwargs.keys():
expiration_date = kwargs['expiration_date']
else:
expiration_date = exp.get_options_expiration(ticker)
expiration_date = expiration_date.date()
settle_datetime = cu.convert_doubledate_2datetime(settle_date)
if 'cal_dte' in kwargs.keys():
cal_dte = kwargs['cal_dte']
else:
cal_dte = (expiration_date-settle_datetime.date()).days
if 'tr_dte' in kwargs.keys():
tr_dte = kwargs['tr_dte']
else:
bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head))
dts = pd.date_range(start=settle_datetime, end=expiration_date, freq=bday_us)
tr_dte = len([x for x in dts if x.to_datetime().date() < expiration_date])
data_vendor_id = 2
now = dt.datetime.now()
con = msu.get_my_sql_connection(**kwargs)
process_output = pco.process_cme_options_4ticker(**kwargs)
if process_output['success']:
settle_frame = process_output['settle_frame']
else:
if 'con' not in kwargs.keys():
con.close()
return
column_names = settle_frame.columns.tolist()
option_type_indx = column_names.index('option_type')
strike_indx = column_names.index('strike')
settle_indx = column_names.index('settle')
volume_indx = column_names.index('volume')
interest_indx = column_names.index('interest')
tuples = [tuple([data_vendor_id, ticker_head, ticker_month_num, ticker_year,
ticker, x[option_type_indx],x[strike_indx],settle_datetime.date(),
cal_dte, tr_dte, now, now,
None if np.isnan(x[settle_indx]) else x[settle_indx],
None if np.isnan(x[volume_indx]) else x[volume_indx],
None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in settle_frame.values]
column_str = "data_vendor_id, ticker_head, ticker_month, ticker_year, ticker, " \
" option_type, strike, price_date, cal_dte, tr_dte, " \
" created_date,last_updated_date, close_price, volume, open_interest"
insert_str = ("%s, " * len(column_str.split(',')))[:-2]
final_str = "REPLACE INTO daily_option_price (%s) VALUES (%s)" % (column_str, insert_str)
msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)
if 'con' not in kwargs.keys():
con.close()
def get_option_greeks(**kwargs):
# risk_free_rate and volatility in whole points not percentage points
# for example enter 0.02 for 2 percent interest rate
# enter 0.25 for 25 percent annual volatility
underlying = kwargs['underlying']
strike = kwargs['strike']
risk_free_rate = kwargs['risk_free_rate']
expiration_date = kwargs['expiration_date']
calculation_date = kwargs['calculation_date']
option_type = kwargs['option_type'].upper()
exercise_type = kwargs['exercise_type'].upper()
dividend_rate = 0
expiration_datetime = cu.convert_doubledate_2datetime(expiration_date)
calculation_datetime = cu.convert_doubledate_2datetime(calculation_date)
expiration_date_obj = ql.Date(expiration_datetime.day, expiration_datetime.month, expiration_datetime.year)
calculation_date_obj = ql.Date(calculation_datetime.day, calculation_datetime.month, calculation_datetime.year)
cal_dte = day_count_obj.dayCount(calculation_date_obj, expiration_date_obj)
nan_greeks = {'option_price': np.NaN,
'implied_vol': np.NaN,
'delta': np.NaN,
'vega': np.NaN,
'theta': np.NaN,
'cal_dte': cal_dte,
'gamma': np.NaN}
#print(underlying)
#print(kwargs['option_price'])
#print(option_type)
#print(exercise_type)
#print(risk_free_rate)
#print(expiration_date)
#print(calculation_date)
#print(strike)
if 'option_price' in kwargs.keys():
if option_type == 'C':
if kwargs['option_price']+strike-underlying <= 10**(-12):
nan_greeks['delta'] = 1
return nan_greeks
elif option_type == 'P':
if kwargs['option_price']-strike+underlying <= 10**(-12):
nan_greeks['delta'] = -1
return nan_greeks
if cal_dte == 0:
if option_type == 'C':
if strike <= underlying:
nan_greeks['delta'] = 1
else:
nan_greeks['delta'] = 0
elif option_type == 'P':
if strike >= underlying:
nan_greeks['delta'] = -1
else:
nan_greeks['delta'] = 0
return nan_greeks
if 'implied_vol' in kwargs.keys():
implied_vol = kwargs['implied_vol']
else:
implied_vol = 0.15
if 'engine_name' in kwargs.keys():
engine_name = kwargs['engine_name']
else:
engine_name = 'baw'
if option_type == 'C':
option_type_obj = ql.Option.Call
elif option_type == 'P':
option_type_obj = ql.Option.Put
ql.Settings.instance().evaluationDate = calculation_date_obj
if exercise_type == 'E':
exercise_obj = ql.EuropeanExercise(expiration_date_obj)
elif exercise_type == 'A':
exercise_obj = ql.AmericanExercise(calculation_date_obj, expiration_date_obj)
#print('years to expitation: ' + str(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj)))
#print('spot: ' + str(underlying/m.exp(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj)*risk_free_rate)))
#underlying_obj = ql.QuoteHandle(ql.SimpleQuote(underlying/m.exp(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj)*risk_free_rate)))
underlying_obj = ql.QuoteHandle(ql.SimpleQuote(underlying))
flat_ts_obj = ql.YieldTermStructureHandle(ql.FlatForward(calculation_date_obj, risk_free_rate, day_count_obj))
dividend_yield_obj = ql.YieldTermStructureHandle(ql.FlatForward(calculation_date_obj, dividend_rate, day_count_obj))
flat_vol_ts_obj = ql.BlackVolTermStructureHandle(ql.BlackConstantVol(calculation_date_obj, calendar_obj, implied_vol, day_count_obj))
#bsm_process = ql.BlackScholesMertonProcess(underlying_obj, dividend_yield_obj, flat_ts_obj, flat_vol_ts_obj)
bsm_process = ql.BlackProcess(underlying_obj, flat_ts_obj, flat_vol_ts_obj)
payoff = ql.PlainVanillaPayoff(option_type_obj, strike)
option_obj = ql.VanillaOption(payoff, exercise_obj)
if engine_name == 'baw':
option_obj.setPricingEngine(ql.BaroneAdesiWhaleyEngine(bsm_process))
elif engine_name == 'fda':
option_obj.setPricingEngine(ql.FDAmericanEngine(bsm_process, 100, 100))
option_price = option_obj.NPV()
if 'option_price' in kwargs.keys():
try:
implied_vol = option_obj.impliedVolatility(targetValue=kwargs['option_price'], process=bsm_process,accuracy=0.00001)
flat_vol_ts_obj = ql.BlackVolTermStructureHandle(ql.BlackConstantVol(calculation_date_obj, calendar_obj, implied_vol, day_count_obj))
#bsm_process = ql.BlackScholesMertonProcess(underlying_obj, dividend_yield_obj, flat_ts_obj, flat_vol_ts_obj)
bsm_process = ql.BlackProcess(underlying_obj, flat_ts_obj, flat_vol_ts_obj)
if engine_name == 'baw':
option_obj.setPricingEngine(ql.BaroneAdesiWhaleyEngine(bsm_process))
elif engine_name == 'fda':
option_obj.setPricingEngine(ql.FDAmericanEngine(bsm_process, 100, 100))
option_price = option_obj.NPV()
except Exception:
return nan_greeks
option_obj = ql.VanillaOption(payoff, ql.EuropeanExercise(expiration_date_obj))
option_obj.setPricingEngine(ql.AnalyticEuropeanEngine(bsm_process))
return {'option_price': option_price,
'implied_vol': implied_vol,
'delta': option_obj.delta(),
'vega': option_obj.vega(),
'theta': option_obj.thetaPerDay(),
'cal_dte': cal_dte,
'gamma': option_obj.gamma()}
def get_curve_pca_report(**kwargs):
ticker_head = kwargs['ticker_head']
date_to = kwargs['date_to']
if 'use_existing_filesQ' in kwargs.keys():
use_existing_filesQ = kwargs['use_existing_filesQ']
else:
use_existing_filesQ = True
output_dir = ts.create_strategy_output_dir(strategy_class='curve_pca', report_date=date_to)
if os.path.isfile(output_dir + '/' + ticker_head + '.pkl') and use_existing_filesQ:
pca_results = pd.read_pickle(output_dir + '/' + ticker_head + '.pkl')
return {'pca_results': pca_results, 'success': True}
date10_years_ago = cu.doubledate_shift(date_to, 10*365)
datetime_to = cu.convert_doubledate_2datetime(date_to)
if ticker_head == 'ED':
num_contracts = 12
rolling_data = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_contracts,
front_tr_dte_limit=10,
date_from=date10_years_ago,
date_to=date_to)
if datetime_to != rolling_data[0].index[-1].to_datetime():
return {'pca_results': pd.DataFrame(), 'success': False}
merged_data = pd.concat(rolling_data, axis=1, join='inner')
total_range = list(range(len(rolling_data)))
index_exclude = [len(total_range)-1]
month_spread = [3]*(len(rolling_data)-1)
elif ticker_head in ['CL', 'B']:
num_monthly_contracts = 18
if ticker_head == 'CL':
num_semiannual_contracts = 6
elif ticker_head == 'B':
num_semiannual_contracts = 7
rolling_data_monthly = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_monthly_contracts,
front_tr_dte_limit=10,
date_from=date10_years_ago,
date_to=date_to)
rolling_data_semiannual = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_semiannual_contracts,
front_tr_dte_limit=10,
date_from=date10_years_ago,
month_separation=6,
date_to=date_to)
if datetime_to != rolling_data_monthly[0].index[-1].to_datetime() or datetime_to != rolling_data_semiannual[0].index[-1].to_datetime():
return {'pca_results': pd.DataFrame(), 'success': False}
rolling_data_merged = pd.concat(rolling_data_semiannual, axis=1)
annual_select = rolling_data_merged['ticker_month'].iloc[-1] % 12 == 0
rolling_data_annual = [rolling_data_semiannual[x] for x in range(len(rolling_data_semiannual)) if annual_select.values[x]]
merged_data = pd.concat(rolling_data_monthly+rolling_data_semiannual+rolling_data_annual, axis=1, join='inner')
total_range = list(range(len(rolling_data_monthly)+len(rolling_data_semiannual)+len(rolling_data_annual)))
index_exclude = [len(rolling_data_monthly)-1,len(rolling_data_monthly)+len(rolling_data_semiannual)-1, len(total_range)-1]
month_spread = [1]*(len(rolling_data_monthly)-1)+[6]*(len(rolling_data_semiannual)-1)+[12]*(len(rolling_data_annual)-1)
yield_raw = [(merged_data['close_price'].ix[:, x]-merged_data['close_price'].ix[:, x+1]) /
merged_data['close_price'].ix[:, x+1] for x in total_range if x not in index_exclude]
yield_merged = pd.concat(yield_raw, axis=1)
yield_data = 100*yield_merged.values
change5_raw = [(merged_data['change5'].ix[:, x]-merged_data['change5'].ix[:, x+1]) for x in total_range
if x not in index_exclude]
change5_merged = pd.concat(change5_raw, axis=1)
change5_data = change5_merged.values
change10_raw = [(merged_data['change10'].ix[:, x]-merged_data['change10'].ix[:, x+1]) for x in total_range
if x not in index_exclude]
change10_merged = pd.concat(change10_raw, axis=1)
change10_data = change10_merged.values
change20_raw = [(merged_data['change20'].ix[:, x]-merged_data['change20'].ix[:, x+1]) for x in total_range
if x not in index_exclude]
change20_merged = pd.concat(change20_raw, axis=1)
change20_data = change20_merged.values
tr_dte_raw = [merged_data['tr_dte'].ix[:, x] for x in total_range if x not in index_exclude]
tr_dte_merged = pd.concat(tr_dte_raw, axis=1)
tr_dte_data = tr_dte_merged.values
ticker_month_raw = [merged_data['ticker_month'].ix[:, x] for x in total_range if x not in index_exclude]
ticker_month_merged = pd.concat(ticker_month_raw, axis=1)
ticker_month_data = ticker_month_merged.values
ticker1_list = [merged_data['ticker'].ix[-1, x] for x in total_range if x not in index_exclude]
ticker2_list = [merged_data['ticker'].ix[-1, x+1] for x in total_range if x not in index_exclude]
price_list = [(merged_data['close_price'].ix[-1, x]-merged_data['close_price'].ix[-1, x+1]) for x in total_range
if x not in index_exclude]
pca_out = stats.get_pca(data_input=yield_data, n_components=2)
residuals = yield_data-pca_out['model_fit']
pca_results = pd.DataFrame.from_items([('ticker1',ticker1_list),
('ticker2',ticker2_list),
('monthSpread',month_spread),
('tr_dte_front', tr_dte_data[-1]),
('ticker_month_front', ticker_month_data[-1]),
('residuals',residuals[-1]),
('price',price_list),
('yield',yield_data[-1]),
('z', (residuals[-1]-residuals.mean(axis=0))/residuals.std(axis=0)),
('factor_load1',pca_out['loadings'][0]),
('factor_load2',pca_out['loadings'][1]),
('change5', change5_data[-1]),
('change10', change10_data[-1]),
('change20', change20_data[-1])])
# notice that this date_to needs to me removed once we are done with backtesting
seasonality_adjustment = fs.get_pca_seasonality_adjustments(ticker_head=ticker_head,date_to=date_to)
pca_results = pd.merge(pca_results, seasonality_adjustment,how='left',on=['monthSpread','ticker_month_front'])
pca_results['z2'] = pca_results['z']-pca_results['z_seasonal_mean']
pca_results['residuals'] = pca_results['residuals'].round(3)
pca_results['yield'] = pca_results['yield'].round(2)
pca_results['z'] = pca_results['z'].round(2)
pca_results['z2'] = pca_results['z2'].round(2)
pca_results['z_seasonal_mean'] = pca_results['z_seasonal_mean'].round(2)
pca_results['factor_load1'] = pca_results['factor_load1'].round(3)
pca_results['factor_load2'] = pca_results['factor_load2'].round(3)
pca_results.to_pickle(output_dir + '/' + ticker_head + '.pkl')
return {'pca_results': pca_results, 'success': True}
def get_aligned_option_indicators(**kwargs):
ticker_list = kwargs['ticker_list']
settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date'])
#print(ticker_list)
if 'num_cal_days_back' in kwargs.keys():
num_cal_days_back = kwargs['num_cal_days_back']
else:
num_cal_days_back = 20*365
settle_datetime_from = settle_datetime-dt.timedelta(num_cal_days_back)
contract_specs_output_list = [cmi.get_contract_specs(x) for x in ticker_list]
ticker_head_list = [x['ticker_head'] for x in contract_specs_output_list]
contract_multiplier_list = [cmi.contract_multiplier[x['ticker_head']] for x in contract_specs_output_list]
cont_indx_list = [x['ticker_year']*100+x['ticker_month_num'] for x in contract_specs_output_list]
month_seperation_list = [cmi.get_month_seperation_from_cont_indx(x,cont_indx_list[0]) for x in cont_indx_list]
if 'option_ticker_indicator_dictionary' in kwargs.keys():
option_ticker_indicator_dictionary = kwargs['option_ticker_indicator_dictionary']
else:
con = msu.get_my_sql_connection(**kwargs)
unique_ticker_heads = list(set(ticker_head_list))
option_ticker_indicator_dictionary = {x: get_option_ticker_indicators(ticker_head=x,
settle_date_to=kwargs['settle_date'],
num_cal_days_back=num_cal_days_back,
con=con) for x in unique_ticker_heads}
if 'con' not in kwargs.keys():
con.close()
option_ticker_indicator_dictionary_final = {ticker_list[x]: option_ticker_indicator_dictionary[ticker_head_list[x]] for x in range(len(ticker_list))}
max_available_settle_list = []
tr_dte_list = []
cal_dte_list = []
imp_vol_list = []
theta_list = []
close2close_vol20_list = []
volume_list = []
open_interest_list = []
for x in range(len(ticker_list)):
ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]]
ticker_data = ticker_data[ticker_data['settle_date'] <= settle_datetime]
option_ticker_indicator_dictionary_final[ticker_list[x]] = ticker_data
ticker_data = ticker_data[ticker_data['ticker'] == ticker_list[x]]
max_available_settle_list.append(ticker_data['settle_date'].iloc[-1])
last_available_settle = min(max_available_settle_list)
for x in range(len(ticker_list)):
ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]]
ticker_data = ticker_data[(ticker_data['ticker'] == ticker_list[x]) & (ticker_data['settle_date'] == last_available_settle)]
tr_dte_list.append(ticker_data['tr_dte'].iloc[0])
cal_dte_list.append(ticker_data['cal_dte'].iloc[0])
imp_vol_list.append(ticker_data['imp_vol'].iloc[0])
theta_list.append(ticker_data['theta'].iloc[0]*contract_multiplier_list[x])
close2close_vol20_list.append(ticker_data['close2close_vol20'].iloc[0])
volume_list.append(ticker_data['volume'].iloc[0])
open_interest_list.append(ticker_data['open_interest'].iloc[0])
current_data = pd.DataFrame.from_items([('ticker',ticker_list),
('tr_dte', tr_dte_list),
('cal_dte', cal_dte_list),
('imp_vol', imp_vol_list),
('theta', theta_list),
('close2close_vol20', close2close_vol20_list),
('volume', volume_list),
('open_interest', open_interest_list)])
current_data['settle_date'] = last_available_settle
current_data.set_index('ticker', drop=True, inplace=True)
current_data = current_data[['settle_date', 'tr_dte', 'cal_dte', 'imp_vol', 'close2close_vol20', 'theta', 'volume', 'open_interest']]
aggregation_method = max([ocu.get_aggregation_method_contracts_back({'ticker_class': x['ticker_class'],
'ticker_head': x['ticker_head']})['aggregation_method'] for x in contract_specs_output_list])
if (current_data['tr_dte'].min() >= 80) and (aggregation_method == 1):
aggregation_method = 3
tr_days_half_band_width_selected = ocu.tr_days_half_band_with[aggregation_method]
data_frame_list = []
ref_tr_dte_list_list = []
for x in range(len(ticker_list)):
ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]]
if ticker_head_list[x] in ['ED', 'E0', 'E2', 'E3', 'E4', 'E5']:
model = 'OU'
else:
model = 'BS'
tr_dte_upper_band = current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected
tr_dte_lower_band = current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected
ref_tr_dte_list = [y for y in cmi.aligned_data_tr_dte_list if y <= tr_dte_upper_band and y>=tr_dte_lower_band]
if len(ref_tr_dte_list) == 0:
return {'hist': [], 'current': [], 'success': False}
if aggregation_method == 12:
aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]],
tr_dte_center=y,
contract_month_letter=contract_specs_output_list[x]['ticker_month_str'],
model=model) for y in ref_tr_dte_list]
ticker_data = ticker_data[ticker_data['ticker_month'] == contract_specs_output_list[x]['ticker_month_num']]
else:
aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]],
tr_dte_center=y,
model=model) for y in ref_tr_dte_list]
aligned_data = [y[(y['trDTE'] >= current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected)&
(y['trDTE'] <= current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected)] for y in aligned_data]
aligned_data = pd.concat(aligned_data)
aligned_data['settle_date'] = pd.to_datetime(aligned_data['settleDates'].astype('str'), format='%Y%m%d')
aligned_data.rename(columns={'TickerYear': 'ticker_year',
'TickerMonth': 'ticker_month',
'trDTE': 'tr_dte',
'calDTE': 'cal_dte',
'impVol': 'imp_vol',
'close2CloseVol20': 'close2close_vol20'}, inplace=True)
aligned_data.sort(['settle_date', 'ticker_year', 'ticker_month'], ascending=[True,True,True],inplace=True)
aligned_data.drop_duplicates(['settle_date','ticker_year','ticker_month'],inplace=True)
aligned_data['old_aligned'] = True
aligned_data = aligned_data[['settle_date','ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'close2close_vol20', 'profit5', 'old_aligned']]
tr_dte_selection = (ticker_data['tr_dte'] >= current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected)&\
(ticker_data['tr_dte'] <= current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected)
ticker_data = ticker_data[tr_dte_selection]
ticker_data['old_aligned'] = False
ticker_data['profit5'] = np.NaN
ticker_data = pd.concat([aligned_data, ticker_data[['settle_date', 'ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'close2close_vol20', 'profit5', 'old_aligned']]])
ticker_data = ticker_data[(ticker_data['settle_date'] <= settle_datetime)&(ticker_data['settle_date'] >= settle_datetime_from)]
ticker_data['cont_indx'] = 100*ticker_data['ticker_year']+ticker_data['ticker_month']
ticker_data['cont_indx_adj'] = [cmi.get_cont_indx_from_month_seperation(y,-month_seperation_list[x]) for y in ticker_data['cont_indx']]
data_frame_list.append(ticker_data)
ref_tr_dte_list_list.append(ref_tr_dte_list)
for x in range(len(ticker_list)):
data_frame_list[x].set_index(['settle_date','cont_indx_adj'], inplace=True,drop=False)
data_frame_list[x]['imp_vol'] = data_frame_list[x]['imp_vol'].astype('float64')
merged_dataframe = pd.concat(data_frame_list, axis=1, join='inner',keys=['c'+ str(x+1) for x in range(len(ticker_list))])
merged_dataframe['abs_tr_dte_diff'] = abs(merged_dataframe['c1']['tr_dte']-tr_dte_list[0])
merged_dataframe['settle_date'] = merged_dataframe['c1']['settle_date']
merged_dataframe.sort(['settle_date', 'abs_tr_dte_diff'], ascending=[True,True], inplace=False)
merged_dataframe.drop_duplicates('settle_date', inplace=True, take_last=False)
merged_dataframe.index = merged_dataframe.index.droplevel(1)
return {'hist': merged_dataframe, 'current': current_data, 'success': True}
def get_vcs_pairs_4date_legacy(**kwargs):
settle_date = kwargs['settle_date']
settle_datetime = cu.convert_doubledate_2datetime(settle_date)
con = msu.get_my_sql_connection(**kwargs)
liquid_options_frame = cl.generate_liquid_options_list_dataframe(settle_date=settle_date,con=con)
contract_specs_output = [cmi.get_contract_specs(x) for x in liquid_options_frame['ticker']]
liquid_options_frame['ticker_head'] = [x['ticker_head'] for x in contract_specs_output]
liquid_options_frame['ticker_month'] = [x['ticker_month_num'] for x in contract_specs_output]
liquid_options_frame['ticker_class'] = [x['ticker_class'] for x in contract_specs_output]
liquid_options_frame['cal_dte'] = [(x-settle_datetime.date()).days for x in liquid_options_frame['expiration_date']]
liquid_options_frame = liquid_options_frame[((liquid_options_frame['ticker_head'] == 'LN')&(liquid_options_frame['cal_dte'] <= 360))|
((liquid_options_frame['ticker_head']=='LC')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']%2==0))|
((liquid_options_frame['ticker_head']=='LC')&(liquid_options_frame['cal_dte']<=40)&
(liquid_options_frame['ticker_month']%2==1))|
((liquid_options_frame['ticker_head']=='ES')&(liquid_options_frame['cal_dte']<=270))|
((liquid_options_frame['ticker_class']=='FX')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month']%3==0))|
((liquid_options_frame['ticker_class']=='FX')&(liquid_options_frame['cal_dte']<=70)&
(liquid_options_frame['ticker_month']%3!=0))|
((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month'].isin([6,12])))|
((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month'].isin([2,4,8,10])))|
((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=70))|
((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month'].isin([7,12])))|
((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month'].isin([1,3,5,9])))|
((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=70))|
((liquid_options_frame['ticker_class']=='Treasury')&(liquid_options_frame['cal_dte']<=180)&
(liquid_options_frame['ticker_month']%3==0))|
((liquid_options_frame['ticker_class']=='Treasury')&(liquid_options_frame['cal_dte']<=70)&
(liquid_options_frame['ticker_month']%3!=0))|
((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=540)&
(liquid_options_frame['ticker_month']==12))|
((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']==7))|
((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month'].isin([3,5,9])))|
((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=40))|
((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=540)&
(liquid_options_frame['ticker_month']==11))|
((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']==7))|
((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month'].isin([1,3,5,8,9])))|
((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=40))|
((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']==12))|
((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month']==7))|
((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=180)&
(liquid_options_frame['ticker_month'].isin([1,3,5,8,9,10])))|
((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=40))|
((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']==12))|
((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month']==7))|
((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=180)&
(liquid_options_frame['ticker_month'].isin([1,3,5,8,9,10])))|
((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=40))|
((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=360)&
(liquid_options_frame['ticker_month']==12))|
((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=270)&
(liquid_options_frame['ticker_month']==7))|
((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=180)&
(liquid_options_frame['ticker_month'].isin([3,5,9])))|
((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=40))|
((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=720)&
(liquid_options_frame['ticker_month']==12))|
((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=540)&
(liquid_options_frame['ticker_month']==6))|
((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=180))|
((liquid_options_frame['ticker_head']=='NG')&(liquid_options_frame['cal_dte']<=360))]
liquid_options_frame.sort(['ticker_head','cal_dte'],ascending=[True,True],inplace=True)
liquid_options_frame['tr_dte'] = [exp.get_days2_expiration(date_to=settle_date,con=con,instrument='options',ticker=x)['tr_dte'] for x in liquid_options_frame['ticker']]
if 'con' not in kwargs.keys():
con.close()
option_frame = liquid_options_frame[liquid_options_frame['tr_dte'] >= 35]
option_frame.reset_index(drop=True,inplace=True)
unique_ticker_heads = option_frame['ticker_head'].unique()
tuples = []
for ticker_head_i in unique_ticker_heads:
ticker_head_data = option_frame[option_frame['ticker_head'] == ticker_head_i]
ticker_head_data.sort('cal_dte', ascending=True, inplace=True)
if len(ticker_head_data.index) >= 2:
for i in range(len(ticker_head_data.index)-1):
for j in range(i+1,len(ticker_head_data.index)):
tuples = tuples + [(ticker_head_data.index[i], ticker_head_data.index[j])]
return pd.DataFrame([(option_frame['ticker'][indx[0]],
option_frame['ticker'][indx[1]],
option_frame['ticker_head'][indx[0]],
option_frame['ticker_class'][indx[0]],
option_frame['tr_dte'][indx[0]],
option_frame['tr_dte'][indx[1]]) for indx in tuples],columns=['ticker1','ticker2','tickerHead','tickerClass','trDte1','trDte2'])
def get_intraday_spread_signals(**kwargs):
ticker_list = kwargs['ticker_list']
date_to = kwargs['date_to']
ticker_list = [x for x in ticker_list if x is not None]
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list]
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
print('-'.join(ticker_list))
if 'tr_dte_list' in kwargs.keys():
tr_dte_list = kwargs['tr_dte_list']
else:
tr_dte_list = [exp.get_days2_expiration(ticker=x,date_to=date_to, instrument='futures')['tr_dte'] for x in ticker_list]
weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list)
if 'aggregation_method' in kwargs.keys() and 'contracts_back' in kwargs.keys():
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
else:
amcb_output = [opUtil.get_aggregation_method_contracts_back(cmi.get_contract_specs(x)) for x in ticker_list]
aggregation_method = max([x['aggregation_method'] for x in amcb_output])
contracts_back = min([x['contracts_back'] for x in amcb_output])
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in list(set(ticker_head_list))}
if 'use_last_as_current' in kwargs.keys():
use_last_as_current = kwargs['use_last_as_current']
else:
use_last_as_current = True
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(date_to,5*365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
if 'num_days_back_4intraday' in kwargs.keys():
num_days_back_4intraday = kwargs['num_days_back_4intraday']
else:
num_days_back_4intraday = 5
contract_multiplier_list = [cmi.contract_multiplier[x] for x in ticker_head_list]
aligned_output = opUtil.get_aligned_futures_data(contract_list=ticker_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=date_to,
futures_data_dictionary=futures_data_dictionary,
use_last_as_current=use_last_as_current)
aligned_data = aligned_output['aligned_data']
current_data = aligned_output['current_data']
spread_weights = weights_output['spread_weights']
portfolio_weights = weights_output['portfolio_weights']
aligned_data['spread'] = 0
aligned_data['spread_pnl_1'] = 0
aligned_data['spread_pnl1'] = 0
spread_settle = 0
last5_years_indx = aligned_data['settle_date']>=datetime5_years_ago
num_contracts = len(ticker_list)
for i in range(num_contracts):
aligned_data['spread'] = aligned_data['spread']+aligned_data['c' + str(i+1)]['close_price']*spread_weights[i]
spread_settle = spread_settle + current_data['c' + str(i+1)]['close_price']*spread_weights[i]
aligned_data['spread_pnl_1'] = aligned_data['spread_pnl_1']+aligned_data['c' + str(i+1)]['change_1']*portfolio_weights[i]*contract_multiplier_list[i]
aligned_data['spread_pnl1'] = aligned_data['spread_pnl1']+aligned_data['c' + str(i+1)]['change1_instant']*portfolio_weights[i]*contract_multiplier_list[i]
aligned_data['spread_normalized'] = aligned_data['spread']/aligned_data['c1']['close_price']
data_last5_years = aligned_data[last5_years_indx]
percentile_vector = stats.get_number_from_quantile(y=data_last5_years['spread_pnl_1'].values,
quantile_list=[1, 15, 85, 99],
clean_num_obs=max(100, round(3*len(data_last5_years.index)/4)))
downside = (percentile_vector[0]+percentile_vector[1])/2
upside = (percentile_vector[2]+percentile_vector[3])/2
date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=x) for x in reversed(range(1,num_days_back_4intraday))]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list,
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date())
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max([cmi.first_trade_hour_minute[x] for x in ticker_head_list])
trade_start_hour = dt.time(9, 30, 0, 0)
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['spread'] = 0
for i in range(num_contracts):
intraday_data['c' + str(i+1), 'mid_p'] = (intraday_data['c' + str(i+1)]['best_bid_p'] +
intraday_data['c' + str(i+1)]['best_ask_p'])/2
intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(i+1)]['mid_p']*spread_weights[i]
unique_settle_dates = intraday_data['settle_date'].unique()
intraday_data['spread1'] = np.nan
for i in range(len(unique_settle_dates)-1):
if (intraday_data['settle_date'] == unique_settle_dates[i]).sum() == \
(intraday_data['settle_date'] == unique_settle_dates[i+1]).sum():
intraday_data.loc[intraday_data['settle_date'] == unique_settle_dates[i],'spread1'] = \
intraday_data['spread'][intraday_data['settle_date'] == unique_settle_dates[i+1]].values
intraday_data = intraday_data[intraday_data['settle_date'].notnull()]
intraday_mean = intraday_data['spread'].mean()
intraday_std = intraday_data['spread'].std()
intraday_data_last2days = intraday_data[intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(date_list[-2]).date()]
intraday_data_yesterday = intraday_data[intraday_data['settle_date'] == cu.convert_doubledate_2datetime(date_list[-1]).date()]
intraday_mean2 = intraday_data_last2days['spread'].mean()
intraday_std2 = intraday_data_last2days['spread'].std()
intraday_mean1 = intraday_data_yesterday['spread'].mean()
intraday_std1 = intraday_data_yesterday['spread'].std()
intraday_z = (spread_settle-intraday_mean)/intraday_std
num_obs_intraday = len(intraday_data.index)
num_obs_intraday_half = round(num_obs_intraday/2)
intraday_tail = intraday_data.tail(num_obs_intraday_half)
num_positives = sum(intraday_tail['spread'] > intraday_data['spread'].mean())
num_negatives = sum(intraday_tail['spread'] < intraday_data['spread'].mean())
recent_trend = 100*(num_positives-num_negatives)/(num_positives+num_negatives)
pnl_frame = ifs.get_pnl_4_date_range(date_to=date_to, num_bus_days_back=20, ticker_list=ticker_list)
if (len(pnl_frame.index)>15)&(pnl_frame['total_pnl'].std() != 0):
historical_sharp = (250**(0.5))*pnl_frame['total_pnl'].mean()/pnl_frame['total_pnl'].std()
else:
historical_sharp = np.nan
return {'downside': downside, 'upside': upside,'intraday_data': intraday_data,
'z': intraday_z,'recent_trend': recent_trend,
'intraday_mean': intraday_mean, 'intraday_std': intraday_std,
'intraday_mean2': intraday_mean2, 'intraday_std2': intraday_std2,
'intraday_mean1': intraday_mean1, 'intraday_std1': intraday_std1,
'aligned_output': aligned_output, 'spread_settle': spread_settle,
'data_last5_years': data_last5_years,'historical_sharp':historical_sharp}
def get_intraday_trend_signals(**kwargs):
ticker = kwargs['ticker']
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
breakout_method = 2
#print(ticker)
ticker_head = cmi.get_contract_specs(ticker)['ticker_head']
contract_multiplier = cmi.contract_multiplier[ticker_head]
ticker_class = cmi.ticker_class[ticker_head]
daily_settles = gfp.get_futures_price_preloaded(ticker=ticker)
daily_settles = daily_settles[daily_settles['settle_date'] <= datetime_to]
daily_settles['ewma10'] = pd.ewma(daily_settles['close_price'], span=10)
daily_settles['ewma50'] = pd.ewma(daily_settles['close_price'], span=50)
if daily_settles['ewma10'].iloc[-1] > daily_settles['ewma50'].iloc[-1]:
long_term_trend = 1
else:
long_term_trend = -1
date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=1)]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=[ticker],
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['hour_minute'] = [100*x.hour+x.minute for x in intraday_data['time_stamp']]
end_hour = cmi.last_trade_hour_minute[ticker_head]
start_hour = cmi.first_trade_hour_minute[ticker_head]
if ticker_class in ['Ag']:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
selected_data = intraday_data.iloc[selection_indx]
selected_data['mid_p'] = (selected_data['c1']['best_bid_p']+selected_data['c1']['best_ask_p'])/2
selected_data['ewma100'] = pd.ewma(selected_data['mid_p'], span=100)
selected_data['ewma25'] = pd.ewma(selected_data['mid_p'], span=25)
selected_data.reset_index(inplace=True,drop=True)
datetime_to = cu.convert_doubledate_2datetime(date_to)
range_start = dt.datetime.combine(datetime_to,dt.time(8,30,0,0))
range_end = dt.datetime.combine(datetime_to,dt.time(9,0,0,0))
first_30_minutes = selected_data[(selected_data['time_stamp'] >= range_start)&
(selected_data['time_stamp'] <= range_end)]
trading_data = selected_data[selected_data['time_stamp'] > range_end]
trading_data_shifted = trading_data.shift(5)
range_min = first_30_minutes['mid_p'].min()
range_max = first_30_minutes['mid_p'].max()
initial_range = range_max-range_min
if breakout_method == 1:
bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)&
(trading_data['mid_p'] < range_max+0.5*initial_range)&
(trading_data_shifted['mid_p']<range_max)&
(trading_data['ewma25'] > range_max)&
(trading_data['mid_p'] > trading_data['ewma100'])]
bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)&
(trading_data['mid_p'] > range_min-0.5*initial_range)&
(trading_data_shifted['mid_p']>range_min)&
(trading_data['ewma25'] < range_min)&
(trading_data['mid_p'] < trading_data['ewma100'])]
elif breakout_method == 2:
bullish_breakout = pd.DataFrame()
bearish_breakout = pd.DataFrame()
if long_term_trend > 0:
bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)&
(trading_data_shifted['mid_p']<range_max)&
(long_term_trend == 1)&
(trading_data['mid_p'] > trading_data['ewma100'])]
elif long_term_trend < 0:
bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)&
(trading_data_shifted['mid_p']>range_min)&
(long_term_trend == -1)&
(trading_data['mid_p'] < trading_data['ewma100'])]
bullish_cross = trading_data[trading_data['mid_p'] > trading_data['ewma100']]
bearish_cross = trading_data[trading_data['mid_p'] < trading_data['ewma100']]
end_of_day_price = trading_data['mid_p'].iloc[-1]
end_of_day_time_stamp = trading_data['time_stamp'].iloc[-1]
valid_bearish_breakoutQ = False
valid_bullish_breakoutQ = False
bearish_breakout_price_entry = np.NaN
bullish_breakout_price_entry = np.NaN
if not bearish_breakout.empty:
if bearish_breakout.index[0]+1 < trading_data.index[-1]:
if trading_data['mid_p'].loc[bearish_breakout.index[0]+1]>range_min-0.5*initial_range:
valid_bearish_breakoutQ = True
bearish_breakout_price_entry = trading_data['mid_p'].loc[bearish_breakout.index[0]+1]
bearish_breakout_time_stamp = trading_data['time_stamp'].loc[bearish_breakout.index[0]+1]
if not bullish_breakout.empty:
if bullish_breakout.index[0]+1<trading_data.index[-1]:
if trading_data['mid_p'].loc[bullish_breakout.index[0]+1]<range_max+0.5*initial_range:
valid_bullish_breakoutQ = True
bullish_breakout_price_entry = trading_data['mid_p'].loc[bullish_breakout.index[0]+1]
bullish_breakout_time_stamp = trading_data['time_stamp'].loc[bullish_breakout.index[0]+1]
stop_loss = (range_max-range_min)*contract_multiplier
pnl_list = []
direction_list = []
entry_time_list = []
exit_time_list = []
entry_price_list = []
exit_price_list = []
daily_trade_no_list = []
exit_type_list = []
ticker_list = []
if valid_bearish_breakoutQ:
direction_list.append(-1)
entry_time_list.append(bearish_breakout_time_stamp)
entry_price_list.append(bearish_breakout_price_entry)
daily_pnl = bearish_breakout_price_entry-end_of_day_price
exit_price = end_of_day_price
exit_type = 'eod'
exit_time = end_of_day_time_stamp
daily_trade_no = 1
#bullish_cross_stop_frame = bullish_cross[(bullish_cross['time_stamp'] > bearish_breakout_time_stamp)]
#if (not bullish_cross_stop_frame.empty) and (bullish_cross_stop_frame.index[0]+1<trading_data.index[-1]):
# daily_pnl = bearish_breakout_price_entry-trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1]
# exit_price = trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1]
# exit_type = 'oso'
# exit_time = trading_data['time_stamp'].loc[bullish_cross_stop_frame.index[0]+1]
#if valid_bullish_breakoutQ:
# if bullish_breakout_time_stamp>bearish_breakout_time_stamp:
# if bullish_breakout_time_stamp<exit_time:
# daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry
# exit_price = bullish_breakout_price_entry
# exit_type = 'fso'
# exit_time = bullish_breakout_time_stamp
# else:
# daily_trade_no = 2
exit_time_list.append(exit_time)
exit_type_list.append(exit_type)
daily_trade_no_list.append(daily_trade_no)
pnl_list.append(daily_pnl)
exit_price_list.append(exit_price)
ticker_list.append(ticker)
if valid_bullish_breakoutQ:
direction_list.append(1)
entry_time_list.append(bullish_breakout_time_stamp)
entry_price_list.append(bullish_breakout_price_entry)
daily_pnl = end_of_day_price-bullish_breakout_price_entry
exit_price = end_of_day_price
exit_type = 'eod'
exit_time = end_of_day_time_stamp
daily_trade_no = 1
bearish_cross_stop_frame = bearish_cross[(bearish_cross['time_stamp'] > bullish_breakout_time_stamp)]
#if (not bearish_cross_stop_frame.empty) and (bearish_cross_stop_frame.index[0]+1 < trading_data.index[-1]):
# daily_pnl = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1]-bullish_breakout_price_entry
# exit_price = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1]
# exit_type = 'oso'
# exit_time = trading_data['time_stamp'].loc[bearish_cross_stop_frame.index[0]+1]
#if valid_bearish_breakoutQ:
# if bearish_breakout_time_stamp>bullish_breakout_time_stamp:
# if bearish_breakout_time_stamp<exit_time:
# daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry
# exit_price = bearish_breakout_price_entry
# exit_type = 'fso'
# exit_time = bearish_breakout_time_stamp
# else:
# daily_trade_no = 2
exit_time_list.append(exit_time)
exit_type_list.append(exit_type)
daily_trade_no_list.append(daily_trade_no)
pnl_list.append(daily_pnl)
exit_price_list.append(exit_price)
ticker_list.append(ticker)
pnl_frame = pd.DataFrame.from_items([('ticker', ticker_list),
('ticker_head',ticker_head),
('direction', direction_list),
('entry_price', entry_price_list),
('exit_price', exit_price_list),
('pnl', pnl_list),
('entry_time', entry_time_list),
('exit_time', exit_time_list),
('exit_type', exit_type_list),
('daily_trade_no', daily_trade_no_list)])
pnl_frame['pnl'] = pnl_frame['pnl']*contract_multiplier
return {'intraday_data': selected_data, 'range_min': range_min, 'range_max':range_max,'pnl_frame':pnl_frame,'stop_loss':stop_loss}
def get_simple_rate(**kwargs):
as_of_date = kwargs["as_of_date"]
date_to = kwargs["date_to"]
if "date_from" in kwargs.keys():
date_from = kwargs["date_from"]
else:
date_from = as_of_date
if "ticker_head" in kwargs.keys():
ticker_head = kwargs["ticker_head"]
else:
ticker_head = "ED"
ta_output_dir = dn.get_dated_directory_extension(folder_date=as_of_date, ext="ta")
file_name = ta_output_dir + "/" + ticker_head + "_interest_curve.pkl"
# print('as_of_date: ' + str(as_of_date) + ', date_to: ' + str(date_to))
if os.path.isfile(file_name):
price_frame = pd.read_pickle(file_name)
if (not os.path.isfile(file_name)) or price_frame.empty:
price_frame = gfp.get_futures_price_preloaded(ticker_head=ticker_head, settle_date=as_of_date)
price_frame = price_frame[price_frame["close_price"].notnull()]
price_frame.sort("tr_dte", ascending=True, inplace=True)
price_frame["exp_date"] = [exp.get_futures_expiration(x) for x in price_frame["ticker"]]
price_frame["implied_rate"] = 100 - price_frame["close_price"]
price_frame.to_pickle(file_name)
if price_frame.empty:
return {
"rate_output": np.NaN,
"price_frame": pd.DataFrame(columns=["ticker", "cal_dte", "exp_date", "implied_rate"]),
}
datetime_to = cu.convert_doubledate_2datetime(date_to)
datetime_from = cu.convert_doubledate_2datetime(date_from)
price_frame_first = price_frame[price_frame["exp_date"] <= datetime_from]
price_frame_middle = price_frame[
(price_frame["exp_date"] > datetime_from) & (price_frame["exp_date"] < datetime_to)
]
if price_frame_middle.empty:
if not price_frame_first.empty:
rate_output = price_frame_first["implied_rate"].iloc[-1] / 100
else:
rate_output = price_frame["implied_rate"].iloc[0] / 100
return {
"rate_output": rate_output,
"price_frame": price_frame[["ticker", "cal_dte", "exp_date", "implied_rate"]],
}
if price_frame_first.empty:
first_rate = price_frame_middle["implied_rate"].iloc[0]
first_period = (price_frame_middle["exp_date"].iloc[0].to_datetime() - datetime_from).days
else:
first_rate = price_frame_first["implied_rate"].iloc[-1]
first_period = (price_frame_middle["exp_date"].iloc[0].to_datetime() - datetime_from).days
last_rate = price_frame_middle["implied_rate"].iloc[-1]
last_period = (datetime_to - price_frame_middle["exp_date"].iloc[-1].to_datetime()).days
middle_discount = [
1
+ (
price_frame_middle["implied_rate"].iloc[x]
* (price_frame_middle["cal_dte"].iloc[x + 1] - price_frame_middle["cal_dte"].iloc[x])
/ 36500
)
for x in range(len(price_frame_middle.index) - 1)
]
total_discount = (
np.prod(np.array(middle_discount))
* (1 + (first_rate * first_period / 36500))
* (1 + (last_rate * last_period / 36500))
)
total_period = (
(price_frame_middle["cal_dte"].iloc[-1] - price_frame_middle["cal_dte"].iloc[0]) + first_period + last_period
)
rate_output = (total_discount - 1) * 365 / total_period
return {"rate_output": rate_output, "price_frame": price_frame[["ticker", "cal_dte", "exp_date", "implied_rate"]]}
def update_futures_price_database_from_cme_file(**kwargs):
ticker_head_list = cmi.cme_futures_tickerhead_list
import time
con = msu.get_my_sql_connection(**kwargs)
if 'settle_date' in kwargs.keys():
run_date = kwargs['settle_date']
else:
run_date = int(time.strftime('%Y%m%d'))
#run_date = 20160225
data_vendor_id = 2
now = datetime.datetime.now()
run_datetime = cu.convert_doubledate_2datetime(run_date)
for ticker_head in ticker_head_list:
#print(ticker_head)
contract_list = []
bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head))
if not exp.is_business_day(double_date=run_date, reference_tickerhead=ticker_head):
continue
cme_output = pcf.process_cme_futures_4tickerhead(ticker_head=ticker_head, report_date=run_date)
settle_frame = cme_output['settle_frame']
for ticker_month in cmi.futures_contract_months[ticker_head]:
ticker_month_num = cmi.letter_month_string.find(ticker_month)+1
max_cal_dte = cmi.get_max_cal_dte(ticker_head=ticker_head, ticker_month=ticker_month_num)
contract_list.extend(cl.get_db_contract_list_filtered(expiration_date_from=run_date,
expiration_date_to=cu.doubledate_shift(run_date, -max_cal_dte),
ticker_head=ticker_head, ticker_month=ticker_month_num, con=con,
instrument='futures'))
contract_frame = pd.DataFrame(contract_list, columns=['symbol_id', 'ticker', 'expiration_date'])
merged_frame = pd.merge(contract_frame,settle_frame, how='inner', on='ticker')
merged_frame.sort('expiration_date', ascending=True, inplace=True)
column_names = merged_frame.columns.tolist()
symbol_id_indx = column_names.index('symbol_id')
ticker_month_indx = column_names.index('ticker_month')
open_indx = column_names.index('open')
high_indx = column_names.index('high')
low_indx = column_names.index('low')
settle_indx = column_names.index('settle')
volume_indx = column_names.index('volume')
interest_indx = column_names.index('interest')
expiration_indx = column_names.index('expiration_date')
dts = pd.date_range(start=run_datetime, end=merged_frame['expiration_date'].iloc[-1], freq=bday_us)
tuples = [tuple([data_vendor_id, x[symbol_id_indx],
ticker_head,
x[ticker_month_indx],
run_datetime.date(),
(x[expiration_indx]-run_datetime.date()).days,
len([y for y in dts if y.to_datetime().date() < x[expiration_indx]]),
now, now,
None if np.isnan(x[open_indx]) else x[open_indx],
None if np.isnan(x[high_indx]) else x[high_indx],
None if np.isnan(x[low_indx]) else x[low_indx],
None if np.isnan(x[settle_indx]) else x[settle_indx],
None if np.isnan(x[volume_indx]) else x[volume_indx],
None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in merged_frame.values]
column_str = "data_vendor_id, symbol_id, ticker_head, ticker_month, price_date,cal_dte, tr_dte, created_date,last_updated_date, open_price, high_price, low_price, close_price, volume, open_interest"
insert_str = ("%s, " * 15)[:-2]
final_str = "REPLACE INTO daily_price (%s) VALUES (%s)" % (column_str, insert_str)
msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con)
if 'con' not in kwargs.keys():
con.close()
def get_vcs_signals_legacy(**kwargs):
aligned_indicators_output = get_aligned_option_indicators_legacy(**kwargs)
if not aligned_indicators_output['success']:
return {'atm_vol_ratio': np.NaN, 'q': np.NaN, 'q2': np.NaN, 'q1': np.NaN, 'q5': np.NaN,
'fwd_vol_q': np.NaN, 'fwd_vol_q2': np.NaN, 'fwd_vol_q1': np.NaN, 'fwd_vol_q5': np.NaN,
'atm_real_vol_ratio': np.NaN, 'q_atm_real_vol_ratio': np.NaN,
'atm_real_vol_ratio_ratio': np.NaN, 'q_atm_real_vol_ratio_ratio': np.NaN,
'tr_dte_diff_percent': np.NaN,'downside': np.NaN, 'upside': np.NaN, 'theta1': np.NaN, 'theta2': np.NaN, 'hist': []}
hist = aligned_indicators_output['hist']
current = aligned_indicators_output['current']
settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date'])
settle_datetime_1year_back = settle_datetime-dt.timedelta(360)
settle_datetime_5year_back = settle_datetime-dt.timedelta(5*360)
hist['atm_vol_ratio'] = hist['c1']['imp_vol']/hist['c2']['imp_vol']
fwd_var = hist['c2']['cal_dte']*(hist['c2']['imp_vol']**2)-hist['c1']['cal_dte']*(hist['c1']['imp_vol']**2)
fwd_vol_sq = fwd_var/(hist['c2']['cal_dte']-hist['c1']['cal_dte'])
fwd_vol_adj = np.sign(fwd_vol_sq)*((abs(fwd_vol_sq)).apply(np.sqrt))
hist['fwd_vol_adj'] = fwd_vol_adj
fwd_var = current['cal_dte'][1]*(current['imp_vol'][1]**2)-current['cal_dte'][0]*(current['imp_vol'][0]**2)
fwd_vol_sq = fwd_var/(current['cal_dte'][1]-current['cal_dte'][0])
fwd_vol_adj = np.sign(fwd_vol_sq)*(np.sqrt(abs(fwd_vol_sq)))
atm_vol_ratio = current['imp_vol'][0]/current['imp_vol'][1]
hist['atm_real_vol_ratio'] = hist['c1']['imp_vol']/hist['c1']['close2close_vol20']
atm_real_vol_ratio = current['imp_vol'][0]/current['close2close_vol20'][0]
hist['atm_real_vol_ratio_ratio'] = (hist['c1']['imp_vol']/hist['c1']['close2close_vol20'])/(hist['c2']['imp_vol']/hist['c2']['close2close_vol20'])
atm_real_vol_ratio_ratio = (current['imp_vol'][0]/current['close2close_vol20'][0])/(current['imp_vol'][0]/current['close2close_vol20'][0])
hist_1year = hist[hist.index >= settle_datetime_1year_back]
hist_5year = hist[hist.index >= settle_datetime_5year_back]
q = stats.get_quantile_from_number({'x': atm_vol_ratio,
'y': hist['atm_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))})
q2 = stats.get_quantile_from_number({'x': atm_vol_ratio, 'y': hist['atm_vol_ratio'].values[-40:], 'clean_num_obs': 30})
q1 = stats.get_quantile_from_number({'x': atm_vol_ratio,
'y': hist_1year['atm_vol_ratio'].values, 'clean_num_obs': max(50, round(3*len(hist_1year.index)/4))})
q5 = stats.get_quantile_from_number({'x': atm_vol_ratio,
'y': hist_5year['atm_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist_5year.index)/4))})
fwd_vol_q = stats.get_quantile_from_number({'x': fwd_vol_adj,
'y': hist['fwd_vol_adj'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))})
fwd_vol_q2 = stats.get_quantile_from_number({'x': fwd_vol_adj,
'y': hist['fwd_vol_adj'].values[-40:], 'clean_num_obs': 30})
fwd_vol_q1 = stats.get_quantile_from_number({'x': fwd_vol_adj,
'y': hist_1year['fwd_vol_adj'].values, 'clean_num_obs': max(50, round(3*len(hist_1year.index)/4))})
fwd_vol_q5 = stats.get_quantile_from_number({'x': fwd_vol_adj,
'y': hist_5year['fwd_vol_adj'].values, 'clean_num_obs': max(100, round(3*len(hist_5year.index)/4))})
q_atm_real_vol_ratio = stats.get_quantile_from_number({'x': atm_real_vol_ratio,
'y': hist['atm_real_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))})
q_atm_real_vol_ratio_ratio = stats.get_quantile_from_number({'x': atm_real_vol_ratio_ratio,
'y': hist['atm_real_vol_ratio_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))})
tr_dte_diff_percent = round(100*(current['tr_dte'][1]-current['tr_dte'][0])/current['tr_dte'][0])
profit5 = hist['c1']['profit5']-hist['c2']['profit5']
clean_indx = profit5.notnull()
clean_data = hist[clean_indx]
if clean_data.empty:
downside = np.NaN
upside = np.NaN
else:
last_available_align_date = clean_data.index[-1]
clean_data = clean_data[clean_data.index >= last_available_align_date-dt.timedelta(5*365)]
profit5 = clean_data['c1']['profit5']-clean_data['c2']['profit5']
percentile_vector = stats.get_number_from_quantile(y=profit5.values,
quantile_list=[1, 15, 85, 99],
clean_num_obs=max(100, round(3*len(profit5.values)/4)))
downside = (percentile_vector[0]+percentile_vector[1])/2
upside = (percentile_vector[2]+percentile_vector[3])/2
return {'atm_vol_ratio': atm_vol_ratio, 'q': q, 'q2': q2, 'q1': q1, 'q5': q5,
'fwd_vol_q': fwd_vol_q, 'fwd_vol_q2': fwd_vol_q2, 'fwd_vol_q1': fwd_vol_q1, 'fwd_vol_q5': fwd_vol_q5,
'atm_real_vol_ratio': atm_real_vol_ratio, 'q_atm_real_vol_ratio': q_atm_real_vol_ratio,
'atm_real_vol_ratio_ratio': atm_real_vol_ratio_ratio, 'q_atm_real_vol_ratio_ratio': q_atm_real_vol_ratio_ratio,
'tr_dte_diff_percent': tr_dte_diff_percent, 'downside': downside, 'upside': upside, 'theta1': current['theta'][0], 'theta2': current['theta'][1], 'hist': hist}
