def get_intraday_outright_covariance(**kwargs):
date_to = kwargs['date_to']
num_days_back_4intraday = 20
liquid_futures_frame = cl.get_liquid_outright_futures_frame(settle_date=date_to)
date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=x) for x in reversed(range(1,num_days_back_4intraday))]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=liquid_futures_frame['ticker'].values,date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['hour_minute'] = [100*x.hour+x.minute for x in intraday_data['time_stamp']]
intraday_data = intraday_data.resample('30min',how='last')
intraday_data = intraday_data[(intraday_data['hour_minute'] >= 830) &
(intraday_data['hour_minute'] <= 1200)]
intraday_data_shifted = intraday_data.shift(1)
selection_indx = intraday_data['hour_minute']-intraday_data_shifted['hour_minute'] > 0
num_contracts = len(liquid_futures_frame.index)
diff_frame = pd.DataFrame()
for i in range(num_contracts):
mid_p = (intraday_data['c' + str(i+1)]['best_bid_p']+
intraday_data['c' + str(i+1)]['best_ask_p'])/2
mid_p_shifted = (intraday_data_shifted['c' + str(i+1)]['best_bid_p']+
intraday_data_shifted['c' + str(i+1)]['best_ask_p'])/2
diff_frame[liquid_futures_frame['ticker_head'].iloc[i]] = (mid_p-mid_p_shifted)*cmi.contract_multiplier[liquid_futures_frame['ticker_head'].iloc[i]]
diff_frame = diff_frame[selection_indx]
return {'cov_matrix': diff_frame.cov(),
'cov_data_integrity': 100*diff_frame.notnull().sum().sum()/(len(diff_frame.columns)*20*6)}
def get_intraday_outright_covariance(**kwargs):
date_to = kwargs['date_to']
num_days_back_4intraday = 20
liquid_futures_frame = cl.get_liquid_outright_futures_frame(
settle_date=date_to)
date_list = [
exp.doubledate_shift_bus_days(double_date=date_to, shift_in_days=x)
for x in reversed(range(1, num_days_back_4intraday))
]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=liquid_futures_frame['ticker'].values,
date_list=date_list)
if len(intraday_data.index) == 0:
return {'cov_matrix': pd.DataFrame(), 'cov_data_integrity': 0}
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['hour_minute'] = [
100 * x.hour + x.minute for x in intraday_data['time_stamp']
]
intraday_data = intraday_data.resample('30min', how='last')
intraday_data = intraday_data[(intraday_data['hour_minute'] >= 830)
& (intraday_data['hour_minute'] <= 1200)]
intraday_data_shifted = intraday_data.shift(1)
selection_indx = intraday_data['hour_minute'] - intraday_data_shifted[
'hour_minute'] > 0
num_contracts = len(liquid_futures_frame.index)
diff_frame = pd.DataFrame()
for i in range(num_contracts):
mid_p = (intraday_data['c' + str(i + 1)]['best_bid_p'] +
intraday_data['c' + str(i + 1)]['best_ask_p']) / 2
mid_p_shifted = (
intraday_data_shifted['c' + str(i + 1)]['best_bid_p'] +
intraday_data_shifted['c' + str(i + 1)]['best_ask_p']) / 2
diff_frame[liquid_futures_frame['ticker_head'].iloc[i]] = (
mid_p - mid_p_shifted) * cmi.contract_multiplier[
liquid_futures_frame['ticker_head'].iloc[i]]
diff_frame = diff_frame[selection_indx]
return {
'cov_matrix':
diff_frame.cov(),
'cov_data_integrity':
100 * diff_frame.notnull().sum().sum() /
(len(diff_frame.columns) * 20 * 6)
}
def get_data4datelist(**kwargs):
ticker_list = kwargs['ticker_list']
date_list = kwargs['date_list']
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list]
if 'spread_weights' in kwargs.keys():
spread_weights = kwargs['spread_weights']
else:
weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list)
spread_weights = weights_output['spread_weights']
num_contracts = len(ticker_list)
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list,
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date())
intraday_data['hour_minute'] = [100*x.hour+x.minute for x in intraday_data['time_stamp']]
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max([cmi.first_trade_hour_minute[x] for x in ticker_head_list])
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['spread'] = 0
for i in range(num_contracts):
intraday_data['c' + str(i+1), 'mid_p'] = (intraday_data['c' + str(i+1)]['best_bid_p'] +
intraday_data['c' + str(i+1)]['best_ask_p'])/2
intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(i+1)]['mid_p']*spread_weights[i]
return intraday_data
def get_intraday_spread_signals(**kwargs):
ticker_list = kwargs['ticker_list']
date_to = kwargs['date_to']
#print(ticker_list)
ticker_list = [x for x in ticker_list if x is not None]
ticker_head_list = [
cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list
]
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
#print('-'.join(ticker_list))
if 'tr_dte_list' in kwargs.keys():
tr_dte_list = kwargs['tr_dte_list']
else:
tr_dte_list = [
exp.get_days2_expiration(ticker=x,
date_to=date_to,
instrument='futures')['tr_dte']
for x in ticker_list
]
if 'aggregation_method' in kwargs.keys(
) and 'contracts_back' in kwargs.keys():
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
else:
amcb_output = [
opUtil.get_aggregation_method_contracts_back(
cmi.get_contract_specs(x)) for x in ticker_list
]
aggregation_method = max(
[x['aggregation_method'] for x in amcb_output])
contracts_back = min([x['contracts_back'] for x in amcb_output])
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {
x: gfp.get_futures_price_preloaded(ticker_head=x)
for x in list(set(ticker_head_list))
}
if 'use_last_as_current' in kwargs.keys():
use_last_as_current = kwargs['use_last_as_current']
else:
use_last_as_current = True
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(date_to, 5 * 365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
if 'num_days_back_4intraday' in kwargs.keys():
num_days_back_4intraday = kwargs['num_days_back_4intraday']
else:
num_days_back_4intraday = 10
contract_multiplier_list = [
cmi.contract_multiplier[x] for x in ticker_head_list
]
aligned_output = opUtil.get_aligned_futures_data(
contract_list=ticker_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=date_to,
futures_data_dictionary=futures_data_dictionary,
use_last_as_current=use_last_as_current)
aligned_data = aligned_output['aligned_data']
current_data = aligned_output['current_data']
if ticker_head_list in fixed_weight_future_spread_list:
weights_output = sutil.get_spread_weights_4contract_list(
ticker_head_list=ticker_head_list)
spread_weights = weights_output['spread_weights']
portfolio_weights = weights_output['portfolio_weights']
else:
regress_output = stats.get_regression_results({
'x':
aligned_data['c2']['change_1'][-60:],
'y':
aligned_data['c1']['change_1'][-60:]
})
spread_weights = [1, -regress_output['beta']]
portfolio_weights = [
1, -regress_output['beta'] * contract_multiplier_list[0] /
contract_multiplier_list[1]
]
aligned_data['spread'] = 0
aligned_data['spread_pnl_1'] = 0
aligned_data['spread_pnl1'] = 0
spread_settle = 0
last5_years_indx = aligned_data['settle_date'] >= datetime5_years_ago
num_contracts = len(ticker_list)
for i in range(num_contracts):
aligned_data['spread'] = aligned_data['spread'] + aligned_data[
'c' + str(i + 1)]['close_price'] * spread_weights[i]
spread_settle = spread_settle + current_data[
'c' + str(i + 1)]['close_price'] * spread_weights[i]
aligned_data[
'spread_pnl_1'] = aligned_data['spread_pnl_1'] + aligned_data[
'c' + str(i + 1)]['change_1'] * portfolio_weights[
i] * contract_multiplier_list[i]
aligned_data[
'spread_pnl1'] = aligned_data['spread_pnl1'] + aligned_data[
'c' + str(i + 1)]['change1_instant'] * portfolio_weights[
i] * contract_multiplier_list[i]
aligned_data['spread_normalized'] = aligned_data['spread'] / aligned_data[
'c1']['close_price']
data_last5_years = aligned_data[last5_years_indx]
percentile_vector = stats.get_number_from_quantile(
y=data_last5_years['spread_pnl_1'].values,
quantile_list=[1, 15, 85, 99],
clean_num_obs=max(100, round(3 * len(data_last5_years.index) / 4)))
downside = (percentile_vector[0] + percentile_vector[1]) / 2
upside = (percentile_vector[2] + percentile_vector[3]) / 2
date_list = [
exp.doubledate_shift_bus_days(double_date=date_to, shift_in_days=x)
for x in reversed(range(1, num_days_back_4intraday))
]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=ticker_list, date_list=date_list)
if len(intraday_data.index) == 0:
return {
'downside': downside,
'upside': upside,
'intraday_data': intraday_data,
'trading_data': intraday_data,
'spread_weight': spread_weights[1],
'portfolio_weight': portfolio_weights[1],
'z': np.nan,
'recent_trend': np.nan,
'intraday_mean10': np.nan,
'intraday_std10': np.nan,
'intraday_mean5': np.nan,
'intraday_std5': np.nan,
'intraday_mean2': np.nan,
'intraday_std2': np.nan,
'intraday_mean1': np.nan,
'intraday_std1': np.nan,
'aligned_output': aligned_output,
'spread_settle': spread_settle,
'data_last5_years': data_last5_years,
'ma_spread_lowL': np.nan,
'ma_spread_highL': np.nan,
'ma_spread_low': np.nan,
'ma_spread_high': np.nan,
'intraday_sharp': np.nan
}
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(
lambda x: x.date())
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max(
[cmi.first_trade_hour_minute[x] for x in ticker_head_list])
trade_start_hour = dt.time(9, 30, 0, 0)
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour))
]
else:
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['spread'] = 0
for i in range(num_contracts):
intraday_data[
'c' + str(i + 1),
'mid_p'] = (intraday_data['c' + str(i + 1)]['best_bid_p'] +
intraday_data['c' + str(i + 1)]['best_ask_p']) / 2
intraday_data['spread'] = intraday_data['spread'] + intraday_data[
'c' + str(i + 1)]['mid_p'] * spread_weights[i]
unique_settle_dates = intraday_data['settle_date'].unique()
intraday_data['spread1'] = np.nan
for i in range(len(unique_settle_dates) - 1):
if (intraday_data['settle_date'] == unique_settle_dates[i]).sum() == \
(intraday_data['settle_date'] == unique_settle_dates[i+1]).sum():
intraday_data.loc[intraday_data['settle_date'] == unique_settle_dates[i],'spread1'] = \
intraday_data['spread'][intraday_data['settle_date'] == unique_settle_dates[i+1]].values
intraday_data = intraday_data[intraday_data['settle_date'].notnull()]
intraday_mean10 = intraday_data['spread'].mean()
intraday_std10 = intraday_data['spread'].std()
intraday_data_last5days = intraday_data[
intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(
date_list[-5]).date()]
intraday_data_last2days = intraday_data[
intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(
date_list[-2]).date()]
intraday_data_yesterday = intraday_data[intraday_data['settle_date'] ==
cu.convert_doubledate_2datetime(
date_list[-1]).date()]
intraday_mean5 = intraday_data_last5days['spread'].mean()
intraday_std5 = intraday_data_last5days['spread'].std()
intraday_mean2 = intraday_data_last2days['spread'].mean()
intraday_std2 = intraday_data_last2days['spread'].std()
intraday_mean1 = intraday_data_yesterday['spread'].mean()
intraday_std1 = intraday_data_yesterday['spread'].std()
intraday_z = (spread_settle - intraday_mean5) / intraday_std5
num_obs_intraday = len(intraday_data.index)
num_obs_intraday_half = round(num_obs_intraday / 2)
intraday_tail = intraday_data.tail(num_obs_intraday_half)
num_positives = sum(
intraday_tail['spread'] > intraday_data['spread'].mean())
num_negatives = sum(
intraday_tail['spread'] < intraday_data['spread'].mean())
if num_positives + num_negatives != 0:
recent_trend = 100 * (num_positives - num_negatives) / (num_positives +
num_negatives)
else:
recent_trend = np.nan
intraday_data_shifted = intraday_data.groupby('settle_date').shift(-60)
intraday_data['spread_shifted'] = intraday_data_shifted['spread']
intraday_data[
'delta60'] = intraday_data['spread_shifted'] - intraday_data['spread']
intraday_data['ewma10'] = pd.ewma(intraday_data['spread'], span=10)
intraday_data['ewma50'] = pd.ewma(intraday_data['spread'], span=50)
intraday_data['ewma200'] = pd.ewma(intraday_data['spread'], span=200)
intraday_data['ma40'] = pd.rolling_mean(intraday_data['spread'], 40)
intraday_data[
'ewma50_spread'] = intraday_data['spread'] - intraday_data['ewma50']
intraday_data[
'ma40_spread'] = intraday_data['spread'] - intraday_data['ma40']
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data['time_stamp'].iloc[x].time() > trade_start_hour)
]
selected_data = intraday_data.iloc[selection_indx]
selected_data['delta60Net'] = (contract_multiplier_list[0] *
selected_data['delta60'] /
spread_weights[0])
selected_data.reset_index(drop=True, inplace=True)
selected_data['proxy_pnl'] = 0
t_cost = cmi.t_cost[ticker_head_list[0]]
ma_spread_low = np.nan
ma_spread_high = np.nan
ma_spread_lowL = np.nan
ma_spread_highL = np.nan
intraday_sharp = np.nan
if sum(selected_data['ma40_spread'].notnull()) > 30:
quantile_list = selected_data['ma40_spread'].quantile([0.1, 0.9])
down_indx = selected_data['ma40_spread'] < quantile_list[0.1]
up_indx = selected_data['ma40_spread'] > quantile_list[0.9]
up_data = selected_data[up_indx]
down_data = selected_data[down_indx]
ma_spread_lowL = quantile_list[0.1]
ma_spread_highL = quantile_list[0.9]
#return {'selected_data':selected_data,'up_data':up_data,'up_indx':up_indx}
selected_data.loc[up_indx,
'proxy_pnl'] = (-up_data['delta60Net'] -
2 * num_contracts * t_cost).values
selected_data.loc[down_indx,
'proxy_pnl'] = (down_data['delta60Net'] -
2 * num_contracts * t_cost).values
short_term_data = selected_data[
selected_data['settle_date'] >= cu.convert_doubledate_2datetime(
date_list[-5]).date()]
if sum(short_term_data['ma40_spread'].notnull()) > 30:
quantile_list = short_term_data['ma40_spread'].quantile([0.1, 0.9])
ma_spread_low = quantile_list[0.1]
ma_spread_high = quantile_list[0.9]
if selected_data['proxy_pnl'].std() != 0:
intraday_sharp = selected_data['proxy_pnl'].mean(
) / selected_data['proxy_pnl'].std()
return {
'downside': downside,
'upside': upside,
'intraday_data': intraday_data,
'trading_data': selected_data,
'spread_weight': spread_weights[1],
'portfolio_weight': portfolio_weights[1],
'z': intraday_z,
'recent_trend': recent_trend,
'intraday_mean10': intraday_mean10,
'intraday_std10': intraday_std10,
'intraday_mean5': intraday_mean5,
'intraday_std5': intraday_std5,
'intraday_mean2': intraday_mean2,
'intraday_std2': intraday_std2,
'intraday_mean1': intraday_mean1,
'intraday_std1': intraday_std1,
'aligned_output': aligned_output,
'spread_settle': spread_settle,
'data_last5_years': data_last5_years,
'ma_spread_lowL': ma_spread_lowL,
'ma_spread_highL': ma_spread_highL,
'ma_spread_low': ma_spread_low,
'ma_spread_high': ma_spread_high,
'intraday_sharp': intraday_sharp
}
def get_ics_signals(**kwargs):
ticker = kwargs['ticker']
#print(ticker)
date_to = kwargs['date_to']
con = msu.get_my_sql_connection(**kwargs)
ticker_list = ticker.split('-')
#print(ticker_list)
ticker_head_list = [
cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list
]
ticker_class = cmi.ticker_class[ticker_head_list[0]]
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {
x: gfp.get_futures_price_preloaded(ticker_head=x)
for x in list(set(ticker_head_list))
}
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(date_to, 5 * 365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
if 'num_days_back_4intraday' in kwargs.keys():
num_days_back_4intraday = kwargs['num_days_back_4intraday']
else:
num_days_back_4intraday = 5
tr_dte_list = [
exp.get_days2_expiration(ticker=x,
date_to=date_to,
instrument='futures',
con=con)['tr_dte'] for x in ticker_list
]
amcb_output = [
opUtil.get_aggregation_method_contracts_back(cmi.get_contract_specs(x))
for x in ticker_list
]
aggregation_method = max([x['aggregation_method'] for x in amcb_output])
contracts_back = min([x['contracts_back'] for x in amcb_output])
contract_multiplier = cmi.contract_multiplier[ticker_head_list[0]]
aligned_output = opUtil.get_aligned_futures_data(
contract_list=ticker_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=date_to,
futures_data_dictionary=futures_data_dictionary,
use_last_as_current=True)
aligned_data = aligned_output['aligned_data']
last5_years_indx = aligned_data['settle_date'] >= datetime5_years_ago
data_last5_years = aligned_data[last5_years_indx]
data_last5_years['spread_pnl_1'] = aligned_data['c1'][
'change_1'] - aligned_data['c2']['change_1']
percentile_vector = stats.get_number_from_quantile(
y=data_last5_years['spread_pnl_1'].values,
quantile_list=[1, 15, 85, 99],
clean_num_obs=max(100, round(3 * len(data_last5_years.index) / 4)))
downside = contract_multiplier * (percentile_vector[0] +
percentile_vector[1]) / 2
upside = contract_multiplier * (percentile_vector[2] +
percentile_vector[3]) / 2
date_list = [
exp.doubledate_shift_bus_days(double_date=date_to, shift_in_days=x)
for x in reversed(range(1, num_days_back_4intraday))
]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=[ticker], date_list=date_list)
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(
lambda x: x.date())
end_hour = cmi.last_trade_hour_minute[ticker_head_list[0]]
start_hour = cmi.first_trade_hour_minute[ticker_head_list[0]]
if ticker_class == 'Ag':
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour))
]
else:
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
intraday_data = intraday_data.iloc[selection_indx]
intraday_mean5 = np.nan
intraday_std5 = np.nan
intraday_mean2 = np.nan
intraday_std2 = np.nan
intraday_mean1 = np.nan
intraday_std1 = np.nan
if len(intraday_data.index) > 0:
intraday_data['mid_p'] = (intraday_data['c1']['best_bid_p'] +
intraday_data['c1']['best_ask_p']) / 2
intraday_mean5 = intraday_data['mid_p'].mean()
intraday_std5 = intraday_data['mid_p'].std()
intraday_data_last2days = intraday_data[
intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(
date_list[-2]).date()]
intraday_data_yesterday = intraday_data[
intraday_data['settle_date'] == cu.convert_doubledate_2datetime(
date_list[-1]).date()]
intraday_mean2 = intraday_data_last2days['mid_p'].mean()
intraday_std2 = intraday_data_last2days['mid_p'].std()
intraday_mean1 = intraday_data_yesterday['mid_p'].mean()
intraday_std1 = intraday_data_yesterday['mid_p'].std()
if 'con' not in kwargs.keys():
con.close()
return {
'downside': downside,
'upside': upside,
'front_tr_dte': tr_dte_list[0],
'intraday_mean5': intraday_mean5,
'intraday_std5': intraday_std5,
'intraday_mean2': intraday_mean2,
'intraday_std2': intraday_std2,
'intraday_mean1': intraday_mean1,
'intraday_std1': intraday_std1
}
def get_intraday_data_4spread(**kwargs):
ticker_list = kwargs['ticker_list']
date_to = kwargs['date_to']
if 'spread_weights' in kwargs.keys():
spread_weights = kwargs['spread_weights']
else:
spread_weights = [1, -1]
if 'num_days_back' in kwargs.keys():
num_days_back = kwargs['num_days_back']
else:
num_days_back = 10
num_contracts = len(ticker_list)
ticker_head_list = [
cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list
]
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
date_list = [
exp.doubledate_shift_bus_days(double_date=date_to, shift_in_days=x)
for x in reversed(range(1, num_days_back))
]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=ticker_list, date_list=date_list)
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['hour_minute'] = [
100 * x.hour + x.minute for x in intraday_data['time_stamp']
]
intraday_data['settle_date'] = [
x.date() for x in intraday_data['time_stamp']
]
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max(
[cmi.first_trade_hour_minute[x] for x in ticker_head_list])
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour))
]
else:
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
selected_data = intraday_data.iloc[selection_indx]
selected_data['spread'] = 0
for i in range(num_contracts):
selected_data[
'c' + str(i + 1),
'mid_p'] = (selected_data['c' + str(i + 1)]['best_bid_p'] +
selected_data['c' + str(i + 1)]['best_ask_p']) / 2
selected_data['spread'] = selected_data['spread'] + selected_data[
'c' + str(i + 1)]['mid_p'] * spread_weights[i]
selected_data_shifted = selected_data.groupby('settle_date').shift(-60)
selected_data['spread_shifted'] = selected_data_shifted['spread']
selected_data[
'delta60'] = selected_data['spread_shifted'] - selected_data['spread']
selected_data['ewma10'] = pd.ewma(selected_data['spread'], span=10)
selected_data['ewma50'] = pd.ewma(selected_data['spread'], span=50)
selected_data['ewma200'] = pd.ewma(selected_data['spread'], span=200)
selected_data['ma40'] = pd.rolling_mean(selected_data['spread'], 40)
selected_data[
'ewma50_spread'] = selected_data['spread'] - selected_data['ewma50']
selected_data[
'ma40_spread'] = selected_data['spread'] - selected_data['ma40']
return selected_data
def get_intraday_data(**kwargs):
ticker = kwargs['ticker']
date_to = kwargs['date_to']
#print(ticker)
if 'num_days_back' in kwargs.keys():
num_days_back = kwargs['num_days_back']
else:
num_days_back = 10
ticker_list = ticker.split('-')
ticker_head = cmi.get_contract_specs(ticker_list[0])['ticker_head']
ticker_class = cmi.ticker_class[ticker_head]
date_list = [
exp.doubledate_shift_bus_days(double_date=date_to, shift_in_days=x)
for x in reversed(range(1, num_days_back))
]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=[ticker], date_list=date_list)
if intraday_data.empty:
return pd.DataFrame()
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['hour_minute'] = [
100 * x.hour + x.minute for x in intraday_data['time_stamp']
]
intraday_data['settle_date'] = [
x.date() for x in intraday_data['time_stamp']
]
end_hour = cmi.last_trade_hour_minute[ticker_head]
start_hour = cmi.first_trade_hour_minute[ticker_head]
if ticker_class in ['Ag']:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour))
]
else:
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
selected_data = intraday_data.iloc[selection_indx]
selected_data['mid_p'] = (selected_data['c1']['best_bid_p'] +
selected_data['c1']['best_ask_p']) / 2
selected_data_shifted = selected_data.groupby('settle_date').shift(-60)
selected_data['mid_p_shifted'] = selected_data_shifted['mid_p']
selected_data['mid_p_delta60'] = selected_data[
'mid_p_shifted'] - selected_data['mid_p']
selected_data['ewma10'] = pd.ewma(selected_data['mid_p'], span=10)
selected_data['ewma50'] = pd.ewma(selected_data['mid_p'], span=50)
selected_data['ma40'] = pd.rolling_mean(selected_data['mid_p'], 40)
selected_data['ma20'] = pd.rolling_mean(selected_data['mid_p'], 20)
selected_data['ma10'] = pd.rolling_mean(selected_data['mid_p'], 10)
selected_data[
'ewma50_spread'] = selected_data['mid_p'] - selected_data['ewma50']
selected_data[
'ma40_spread'] = selected_data['mid_p'] - selected_data['ma40']
selected_data[
'ma20_spread'] = selected_data['mid_p'] - selected_data['ma20']
selected_data[
'ma10_spread'] = selected_data['mid_p'] - selected_data['ma10']
return selected_data
def backtest_continuous_ics(**kwargs):
intraday_spreads = kwargs['intraday_spreads']
trade_id = kwargs['trade_id']
date_list = kwargs['date_list']
ticker = intraday_spreads.iloc[trade_id]['ticker']
num_contracts = 2
ticker_list = ticker.split('-')
ticker_head_list = [
cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list
]
ticker_class = cmi.ticker_class[ticker_head_list[0]]
contract_multiplier_list = [
cmi.contract_multiplier[x] for x in ticker_head_list
]
t_cost = cmi.t_cost[ticker_head_list[0]]
intraday_data = opUtil.get_aligned_futures_data_intraday(
contract_list=[ticker], date_list=date_list)
intraday_data['time_stamp'] = [
x.to_datetime() for x in intraday_data.index
]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(
lambda x: x.date())
intraday_data['hour_minute'] = [
100 * x.hour + x.minute for x in intraday_data['time_stamp']
]
end_hour = cmi.last_trade_hour_minute[ticker_head_list[0]]
start_hour = cmi.first_trade_hour_minute[ticker_head_list[0]]
if ticker_class == 'Ag':
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour) and
(intraday_data['time_stamp'].iloc[x].time() >= start_hour))
]
else:
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
intraday_data = intraday_data.iloc[selection_indx]
start_hour = dt.time(9, 0, 0, 0)
end_hour = dt.time(12, 55, 0, 0)
selection_indx = [
x for x in range(len(intraday_data.index))
if (intraday_data.index[x].to_datetime().time() < end_hour) and (
intraday_data.index[x].to_datetime().time() >= start_hour)
]
intraday_data = intraday_data.iloc[selection_indx]
mean5 = intraday_spreads.iloc[trade_id]['intraday_mean5']
std5 = intraday_spreads.iloc[trade_id]['intraday_std5']
mean2 = intraday_spreads.iloc[trade_id]['intraday_mean2']
std2 = intraday_spreads.iloc[trade_id]['intraday_std2']
mean1 = intraday_spreads.iloc[trade_id]['intraday_mean1']
std1 = intraday_spreads.iloc[trade_id]['intraday_std1']
long_qty = -5000 / intraday_spreads.iloc[trade_id]['downside']
short_qty = -5000 / intraday_spreads.iloc[trade_id]['upside']
intraday_data['spread'] = (intraday_data['c1']['best_bid_p'] +
intraday_data['c1']['best_ask_p']) / 2
intraday_data['z5'] = (intraday_data['spread'] - mean5) / std5
intraday_data['z1'] = (intraday_data['spread'] - mean1) / std1
intraday_data['z2'] = (intraday_data['spread'] - mean2) / std2
intraday_data['z6'] = (intraday_data['spread'] - mean1) / std5
entry_data = intraday_data[intraday_data['settle_date'] ==
cu.convert_doubledate_2datetime(
date_list[-2]).date()]
exit_data = intraday_data[intraday_data['settle_date'] ==
cu.convert_doubledate_2datetime(
date_list[-1]).date()]
exit_morning_data = exit_data[(exit_data['hour_minute'] >= 930)
& (exit_data['hour_minute'] <= 1000)]
exit_afternoon_data = exit_data[(exit_data['hour_minute'] >= 1230)
& (exit_data['hour_minute'] <= 1300)]
entry_data['pnl_long_morning'] = long_qty * (
exit_morning_data['spread'].mean() -
entry_data['spread']) * contract_multiplier_list[0]
entry_data['pnl_long_morning_wc'] = entry_data[
'pnl_long_morning'] - 2 * t_cost * long_qty * num_contracts
entry_data['pnl_short_morning'] = short_qty * (
exit_morning_data['spread'].mean() -
entry_data['spread']) * contract_multiplier_list[0]
entry_data['pnl_short_morning_wc'] = entry_data[
'pnl_short_morning'] - 2 * t_cost * abs(short_qty) * num_contracts
entry_data['pnl_long_afternoon'] = long_qty * (
exit_afternoon_data['spread'].mean() -
entry_data['spread']) * contract_multiplier_list[0]
entry_data['pnl_long_afternoon_wc'] = entry_data[
'pnl_long_afternoon'] - 2 * t_cost * long_qty * num_contracts
entry_data['pnl_short_afternoon'] = short_qty * (
exit_afternoon_data['spread'].mean() -
entry_data['spread']) * contract_multiplier_list[0]
entry_data['pnl_short_afternoon_wc'] = entry_data[
'pnl_short_afternoon'] - 2 * t_cost * abs(short_qty) * num_contracts
entry_data['pnl_morning_wc'] = 0
entry_data['pnl_afternoon_wc'] = 0
long_indx = entry_data['z1'] < 0
short_indx = entry_data['z1'] > 0
entry_data.loc[long_indx, 'pnl_morning_wc'] = entry_data.loc[
long_indx, 'pnl_long_morning_wc'].values
entry_data.loc[short_indx > 0, 'pnl_morning_wc'] = entry_data.loc[
short_indx > 0, 'pnl_short_morning_wc'].values
entry_data.loc[long_indx < 0, 'pnl_afternoon_wc'] = entry_data.loc[
long_indx < 0, 'pnl_long_afternoon_wc'].values
entry_data.loc[short_indx > 0, 'pnl_afternoon_wc'] = entry_data.loc[
short_indx > 0, 'pnl_short_afternoon_wc'].values
return entry_data
def get_intraday_trend_signals(**kwargs):
ticker = kwargs['ticker']
date_to = kwargs['date_to']
datetime_to = cu.convert_doubledate_2datetime(date_to)
breakout_method = 2
#print(ticker)
ticker_head = cmi.get_contract_specs(ticker)['ticker_head']
contract_multiplier = cmi.contract_multiplier[ticker_head]
ticker_class = cmi.ticker_class[ticker_head]
daily_settles = gfp.get_futures_price_preloaded(ticker=ticker)
daily_settles = daily_settles[daily_settles['settle_date'] <= datetime_to]
daily_settles['ewma10'] = pd.ewma(daily_settles['close_price'], span=10)
daily_settles['ewma50'] = pd.ewma(daily_settles['close_price'], span=50)
if daily_settles['ewma10'].iloc[-1] > daily_settles['ewma50'].iloc[-1]:
long_term_trend = 1
else:
long_term_trend = -1
date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=1)]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=[ticker],
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['hour_minute'] = [100*x.hour+x.minute for x in intraday_data['time_stamp']]
end_hour = cmi.last_trade_hour_minute[ticker_head]
start_hour = cmi.first_trade_hour_minute[ticker_head]
if ticker_class in ['Ag']:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
selected_data = intraday_data.iloc[selection_indx]
selected_data['mid_p'] = (selected_data['c1']['best_bid_p']+selected_data['c1']['best_ask_p'])/2
selected_data['ewma100'] = pd.ewma(selected_data['mid_p'], span=100)
selected_data['ewma25'] = pd.ewma(selected_data['mid_p'], span=25)
selected_data.reset_index(inplace=True,drop=True)
datetime_to = cu.convert_doubledate_2datetime(date_to)
range_start = dt.datetime.combine(datetime_to,dt.time(8,30,0,0))
range_end = dt.datetime.combine(datetime_to,dt.time(9,0,0,0))
first_30_minutes = selected_data[(selected_data['time_stamp'] >= range_start)&
(selected_data['time_stamp'] <= range_end)]
trading_data = selected_data[selected_data['time_stamp'] > range_end]
trading_data_shifted = trading_data.shift(5)
range_min = first_30_minutes['mid_p'].min()
range_max = first_30_minutes['mid_p'].max()
initial_range = range_max-range_min
if breakout_method == 1:
bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)&
(trading_data['mid_p'] < range_max+0.5*initial_range)&
(trading_data_shifted['mid_p']<range_max)&
(trading_data['ewma25'] > range_max)&
(trading_data['mid_p'] > trading_data['ewma100'])]
bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)&
(trading_data['mid_p'] > range_min-0.5*initial_range)&
(trading_data_shifted['mid_p']>range_min)&
(trading_data['ewma25'] < range_min)&
(trading_data['mid_p'] < trading_data['ewma100'])]
elif breakout_method == 2:
bullish_breakout = pd.DataFrame()
bearish_breakout = pd.DataFrame()
if long_term_trend > 0:
bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)&
(trading_data_shifted['mid_p']<range_max)&
(long_term_trend == 1)&
(trading_data['mid_p'] > trading_data['ewma100'])]
elif long_term_trend < 0:
bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)&
(trading_data_shifted['mid_p']>range_min)&
(long_term_trend == -1)&
(trading_data['mid_p'] < trading_data['ewma100'])]
bullish_cross = trading_data[trading_data['mid_p'] > trading_data['ewma100']]
bearish_cross = trading_data[trading_data['mid_p'] < trading_data['ewma100']]
end_of_day_price = trading_data['mid_p'].iloc[-1]
end_of_day_time_stamp = trading_data['time_stamp'].iloc[-1]
valid_bearish_breakoutQ = False
valid_bullish_breakoutQ = False
bearish_breakout_price_entry = np.NaN
bullish_breakout_price_entry = np.NaN
if not bearish_breakout.empty:
if bearish_breakout.index[0]+1 < trading_data.index[-1]:
if trading_data['mid_p'].loc[bearish_breakout.index[0]+1]>range_min-0.5*initial_range:
valid_bearish_breakoutQ = True
bearish_breakout_price_entry = trading_data['mid_p'].loc[bearish_breakout.index[0]+1]
bearish_breakout_time_stamp = trading_data['time_stamp'].loc[bearish_breakout.index[0]+1]
if not bullish_breakout.empty:
if bullish_breakout.index[0]+1<trading_data.index[-1]:
if trading_data['mid_p'].loc[bullish_breakout.index[0]+1]<range_max+0.5*initial_range:
valid_bullish_breakoutQ = True
bullish_breakout_price_entry = trading_data['mid_p'].loc[bullish_breakout.index[0]+1]
bullish_breakout_time_stamp = trading_data['time_stamp'].loc[bullish_breakout.index[0]+1]
stop_loss = (range_max-range_min)*contract_multiplier
pnl_list = []
direction_list = []
entry_time_list = []
exit_time_list = []
entry_price_list = []
exit_price_list = []
daily_trade_no_list = []
exit_type_list = []
ticker_list = []
if valid_bearish_breakoutQ:
direction_list.append(-1)
entry_time_list.append(bearish_breakout_time_stamp)
entry_price_list.append(bearish_breakout_price_entry)
daily_pnl = bearish_breakout_price_entry-end_of_day_price
exit_price = end_of_day_price
exit_type = 'eod'
exit_time = end_of_day_time_stamp
daily_trade_no = 1
#bullish_cross_stop_frame = bullish_cross[(bullish_cross['time_stamp'] > bearish_breakout_time_stamp)]
#if (not bullish_cross_stop_frame.empty) and (bullish_cross_stop_frame.index[0]+1<trading_data.index[-1]):
# daily_pnl = bearish_breakout_price_entry-trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1]
# exit_price = trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1]
# exit_type = 'oso'
# exit_time = trading_data['time_stamp'].loc[bullish_cross_stop_frame.index[0]+1]
#if valid_bullish_breakoutQ:
# if bullish_breakout_time_stamp>bearish_breakout_time_stamp:
# if bullish_breakout_time_stamp<exit_time:
# daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry
# exit_price = bullish_breakout_price_entry
# exit_type = 'fso'
# exit_time = bullish_breakout_time_stamp
# else:
# daily_trade_no = 2
exit_time_list.append(exit_time)
exit_type_list.append(exit_type)
daily_trade_no_list.append(daily_trade_no)
pnl_list.append(daily_pnl)
exit_price_list.append(exit_price)
ticker_list.append(ticker)
if valid_bullish_breakoutQ:
direction_list.append(1)
entry_time_list.append(bullish_breakout_time_stamp)
entry_price_list.append(bullish_breakout_price_entry)
daily_pnl = end_of_day_price-bullish_breakout_price_entry
exit_price = end_of_day_price
exit_type = 'eod'
exit_time = end_of_day_time_stamp
daily_trade_no = 1
bearish_cross_stop_frame = bearish_cross[(bearish_cross['time_stamp'] > bullish_breakout_time_stamp)]
#if (not bearish_cross_stop_frame.empty) and (bearish_cross_stop_frame.index[0]+1 < trading_data.index[-1]):
# daily_pnl = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1]-bullish_breakout_price_entry
# exit_price = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1]
# exit_type = 'oso'
# exit_time = trading_data['time_stamp'].loc[bearish_cross_stop_frame.index[0]+1]
#if valid_bearish_breakoutQ:
# if bearish_breakout_time_stamp>bullish_breakout_time_stamp:
# if bearish_breakout_time_stamp<exit_time:
# daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry
# exit_price = bearish_breakout_price_entry
# exit_type = 'fso'
# exit_time = bearish_breakout_time_stamp
# else:
# daily_trade_no = 2
exit_time_list.append(exit_time)
exit_type_list.append(exit_type)
daily_trade_no_list.append(daily_trade_no)
pnl_list.append(daily_pnl)
exit_price_list.append(exit_price)
ticker_list.append(ticker)
pnl_frame = pd.DataFrame.from_items([('ticker', ticker_list),
('ticker_head',ticker_head),
('direction', direction_list),
('entry_price', entry_price_list),
('exit_price', exit_price_list),
('pnl', pnl_list),
('entry_time', entry_time_list),
('exit_time', exit_time_list),
('exit_type', exit_type_list),
('daily_trade_no', daily_trade_no_list)])
pnl_frame['pnl'] = pnl_frame['pnl']*contract_multiplier
return {'intraday_data': selected_data, 'range_min': range_min, 'range_max':range_max,'pnl_frame':pnl_frame,'stop_loss':stop_loss}
def get_intraday_spread_signals(**kwargs):
ticker_list = kwargs['ticker_list']
date_to = kwargs['date_to']
ticker_list = [x for x in ticker_list if x is not None]
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list]
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
print('-'.join(ticker_list))
if 'tr_dte_list' in kwargs.keys():
tr_dte_list = kwargs['tr_dte_list']
else:
tr_dte_list = [exp.get_days2_expiration(ticker=x,date_to=date_to, instrument='futures')['tr_dte'] for x in ticker_list]
weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list)
if 'aggregation_method' in kwargs.keys() and 'contracts_back' in kwargs.keys():
aggregation_method = kwargs['aggregation_method']
contracts_back = kwargs['contracts_back']
else:
amcb_output = [opUtil.get_aggregation_method_contracts_back(cmi.get_contract_specs(x)) for x in ticker_list]
aggregation_method = max([x['aggregation_method'] for x in amcb_output])
contracts_back = min([x['contracts_back'] for x in amcb_output])
if 'futures_data_dictionary' in kwargs.keys():
futures_data_dictionary = kwargs['futures_data_dictionary']
else:
futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in list(set(ticker_head_list))}
if 'use_last_as_current' in kwargs.keys():
use_last_as_current = kwargs['use_last_as_current']
else:
use_last_as_current = True
if 'datetime5_years_ago' in kwargs.keys():
datetime5_years_ago = kwargs['datetime5_years_ago']
else:
date5_years_ago = cu.doubledate_shift(date_to,5*365)
datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago)
if 'num_days_back_4intraday' in kwargs.keys():
num_days_back_4intraday = kwargs['num_days_back_4intraday']
else:
num_days_back_4intraday = 5
contract_multiplier_list = [cmi.contract_multiplier[x] for x in ticker_head_list]
aligned_output = opUtil.get_aligned_futures_data(contract_list=ticker_list,
tr_dte_list=tr_dte_list,
aggregation_method=aggregation_method,
contracts_back=contracts_back,
date_to=date_to,
futures_data_dictionary=futures_data_dictionary,
use_last_as_current=use_last_as_current)
aligned_data = aligned_output['aligned_data']
current_data = aligned_output['current_data']
spread_weights = weights_output['spread_weights']
portfolio_weights = weights_output['portfolio_weights']
aligned_data['spread'] = 0
aligned_data['spread_pnl_1'] = 0
aligned_data['spread_pnl1'] = 0
spread_settle = 0
last5_years_indx = aligned_data['settle_date']>=datetime5_years_ago
num_contracts = len(ticker_list)
for i in range(num_contracts):
aligned_data['spread'] = aligned_data['spread']+aligned_data['c' + str(i+1)]['close_price']*spread_weights[i]
spread_settle = spread_settle + current_data['c' + str(i+1)]['close_price']*spread_weights[i]
aligned_data['spread_pnl_1'] = aligned_data['spread_pnl_1']+aligned_data['c' + str(i+1)]['change_1']*portfolio_weights[i]*contract_multiplier_list[i]
aligned_data['spread_pnl1'] = aligned_data['spread_pnl1']+aligned_data['c' + str(i+1)]['change1_instant']*portfolio_weights[i]*contract_multiplier_list[i]
aligned_data['spread_normalized'] = aligned_data['spread']/aligned_data['c1']['close_price']
data_last5_years = aligned_data[last5_years_indx]
percentile_vector = stats.get_number_from_quantile(y=data_last5_years['spread_pnl_1'].values,
quantile_list=[1, 15, 85, 99],
clean_num_obs=max(100, round(3*len(data_last5_years.index)/4)))
downside = (percentile_vector[0]+percentile_vector[1])/2
upside = (percentile_vector[2]+percentile_vector[3])/2
date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=x) for x in reversed(range(1,num_days_back_4intraday))]
date_list.append(date_to)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list,
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date())
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max([cmi.first_trade_hour_minute[x] for x in ticker_head_list])
trade_start_hour = dt.time(9, 30, 0, 0)
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['spread'] = 0
for i in range(num_contracts):
intraday_data['c' + str(i+1), 'mid_p'] = (intraday_data['c' + str(i+1)]['best_bid_p'] +
intraday_data['c' + str(i+1)]['best_ask_p'])/2
intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(i+1)]['mid_p']*spread_weights[i]
unique_settle_dates = intraday_data['settle_date'].unique()
intraday_data['spread1'] = np.nan
for i in range(len(unique_settle_dates)-1):
if (intraday_data['settle_date'] == unique_settle_dates[i]).sum() == \
(intraday_data['settle_date'] == unique_settle_dates[i+1]).sum():
intraday_data.loc[intraday_data['settle_date'] == unique_settle_dates[i],'spread1'] = \
intraday_data['spread'][intraday_data['settle_date'] == unique_settle_dates[i+1]].values
intraday_data = intraday_data[intraday_data['settle_date'].notnull()]
intraday_mean = intraday_data['spread'].mean()
intraday_std = intraday_data['spread'].std()
intraday_data_last2days = intraday_data[intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(date_list[-2]).date()]
intraday_data_yesterday = intraday_data[intraday_data['settle_date'] == cu.convert_doubledate_2datetime(date_list[-1]).date()]
intraday_mean2 = intraday_data_last2days['spread'].mean()
intraday_std2 = intraday_data_last2days['spread'].std()
intraday_mean1 = intraday_data_yesterday['spread'].mean()
intraday_std1 = intraday_data_yesterday['spread'].std()
intraday_z = (spread_settle-intraday_mean)/intraday_std
num_obs_intraday = len(intraday_data.index)
num_obs_intraday_half = round(num_obs_intraday/2)
intraday_tail = intraday_data.tail(num_obs_intraday_half)
num_positives = sum(intraday_tail['spread'] > intraday_data['spread'].mean())
num_negatives = sum(intraday_tail['spread'] < intraday_data['spread'].mean())
recent_trend = 100*(num_positives-num_negatives)/(num_positives+num_negatives)
pnl_frame = ifs.get_pnl_4_date_range(date_to=date_to, num_bus_days_back=20, ticker_list=ticker_list)
if (len(pnl_frame.index)>15)&(pnl_frame['total_pnl'].std() != 0):
historical_sharp = (250**(0.5))*pnl_frame['total_pnl'].mean()/pnl_frame['total_pnl'].std()
else:
historical_sharp = np.nan
return {'downside': downside, 'upside': upside,'intraday_data': intraday_data,
'z': intraday_z,'recent_trend': recent_trend,
'intraday_mean': intraday_mean, 'intraday_std': intraday_std,
'intraday_mean2': intraday_mean2, 'intraday_std2': intraday_std2,
'intraday_mean1': intraday_mean1, 'intraday_std1': intraday_std1,
'aligned_output': aligned_output, 'spread_settle': spread_settle,
'data_last5_years': data_last5_years,'historical_sharp':historical_sharp}
def backtest_ifs_4date(**kwargs):
report_date = kwargs['report_date']
output_dir = ts.create_strategy_output_dir(strategy_class='ifs', report_date=report_date)
#if os.path.isfile(output_dir + '/backtest_results.pkl'):
# intraday_spreads = pd.read_pickle(output_dir + '/backtest_results.pkl')
# return intraday_spreads
start_hour = dt.time(9, 0, 0, 0)
end_hour = dt.time(12, 55, 0, 0)
sheet_output = ifs.generate_ifs_sheet_4date(date_to=report_date)
intraday_spreads = sheet_output['intraday_spreads']
intraday_spreads['pnl1'] = 0
intraday_spreads['pnl2'] = 0
intraday_spreads['pnl5'] = 0
intraday_spreads['pnl6'] = 0
intraday_spreads['pnl7'] = 0
intraday_spreads['pnl1_wc'] = 0
intraday_spreads['pnl2_wc'] = 0
intraday_spreads['pnl5_wc'] = 0
intraday_spreads['pnl6_wc'] = 0
intraday_spreads['pnl7_wc'] = 0
intraday_spreads['report_date'] = report_date
intraday_spreads['spread_description'] = intraday_spreads.apply(lambda x: x['ticker_head1']+ '_' +x['ticker_head2'] if x['ticker_head3'] is None else x['ticker_head1']+ '_' +x['ticker_head2'] + '_' + x['ticker_head3'] ,axis=1)
intraday_spreads['min_volume'] = intraday_spreads.apply(lambda x: min(x['volume1'],x['volume2']) if x['ticker_head3'] is None else min(x['volume1'],x['volume2'],x['volume3']) ,axis=1)
intraday_spreads.sort(['spread_description','min_volume'],ascending=[True, False],inplace=True)
intraday_spreads.drop_duplicates('spread_description',inplace=True)
intraday_spreads = intraday_spreads[intraday_spreads['hs']>-1]
intraday_spreads.reset_index(drop=True,inplace=True)
date_list = [exp.doubledate_shift_bus_days(double_date=report_date, shift_in_days=x) for x in [-1,-2]]
for i in range(len(intraday_spreads.index)):
#print(i)
ticker_list = [intraday_spreads.iloc[i]['contract1'],intraday_spreads.iloc[i]['contract2'],intraday_spreads.iloc[i]['contract3']]
ticker_list = [x for x in ticker_list if x is not None]
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list]
num_contracts = len(ticker_list)
weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list)
contract_multiplier_list = [cmi.contract_multiplier[x] for x in ticker_head_list]
spread_weights = weights_output['spread_weights']
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list,
date_list=date_list)
intraday_data['spread'] = 0
for j in range(num_contracts):
intraday_data['c' + str(j+1), 'mid_p'] = (intraday_data['c' + str(j+1)]['best_bid_p'] +
intraday_data['c' + str(j+1)]['best_ask_p'])/2
intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(j+1)]['mid_p']*spread_weights[j]
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date())
unique_settle_dates = intraday_data['settle_date'].unique()
intraday_data['spread1'] = np.nan
if len(unique_settle_dates)<2:
continue
final_spread_price = np.mean(intraday_data['spread'][(intraday_data['settle_date'] == unique_settle_dates[1])])
intraday_data = intraday_data[(intraday_data['settle_date'] == unique_settle_dates[0])]
intraday_data['spread_diff'] = contract_multiplier_list[0]*(final_spread_price-intraday_data['spread'])/spread_weights[0]
mean5 = intraday_spreads.iloc[i]['mean']
std5 = intraday_spreads.iloc[i]['std']
mean1 = intraday_spreads.iloc[i]['mean1']
std1 = intraday_spreads.iloc[i]['std1']
mean2 = intraday_spreads.iloc[i]['mean2']
std2 = intraday_spreads.iloc[i]['std2']
long_qty = -5000/intraday_spreads.iloc[i]['downside']
short_qty = -5000/intraday_spreads.iloc[i]['upside']
intraday_data['z5'] = (intraday_data['spread']-mean5)/std5
intraday_data['z1'] = (intraday_data['spread']-mean1)/std1
intraday_data['z2'] = (intraday_data['spread']-mean2)/std2
intraday_data['z6'] = (intraday_data['spread']-mean1)/std5
intraday_data11 = intraday_data[intraday_data['z1']>1]
intraday_data1_1 = intraday_data[intraday_data['z1']<-1]
if intraday_data1_1.empty:
pnl1_1 = 0
pnl1_1_wc = 0
else:
pnl1_1 = long_qty*intraday_data1_1['spread_diff'].mean()
pnl1_1_wc = pnl1_1 - 2*2*long_qty*num_contracts
if intraday_data11.empty:
pnl11 = 0
pnl11_wc = 0
else:
pnl11 = short_qty*intraday_data11['spread_diff'].mean()
pnl11_wc = pnl11 + 2*2*short_qty*num_contracts
intraday_data21 = intraday_data[intraday_data['z2']>1]
intraday_data2_1 = intraday_data[intraday_data['z2']<-1]
if intraday_data2_1.empty:
pnl2_1 = 0
pnl2_1_wc = 0
else:
pnl2_1 = long_qty*intraday_data2_1['spread_diff'].mean()
pnl2_1_wc = pnl2_1 - 2*2*long_qty*num_contracts
if intraday_data21.empty:
pnl21 = 0
pnl21_wc = 0
else:
pnl21 = short_qty*intraday_data21['spread_diff'].mean()
pnl21_wc = pnl21 + 2*2*short_qty*num_contracts
intraday_data51 = intraday_data[intraday_data['z5']>1]
intraday_data5_1 = intraday_data[intraday_data['z5']<-1]
if intraday_data5_1.empty:
pnl5_1 = 0
pnl5_1_wc = 0
else:
pnl5_1 = long_qty*intraday_data5_1['spread_diff'].mean()
pnl5_1_wc = pnl5_1 - 2*2*long_qty*num_contracts
if intraday_data51.empty:
pnl51 = 0
pnl51_wc = 0
else:
pnl51 = short_qty*intraday_data51['spread_diff'].mean()
pnl51_wc = pnl51 + 2*2*short_qty*num_contracts
intraday_data61 = intraday_data[intraday_data['z6']>0.25]
intraday_data6_1 = intraday_data[intraday_data['z6']<-0.25]
if intraday_data6_1.empty:
pnl6_1 = 0
pnl6_1_wc = 0
else:
pnl6_1 = long_qty*intraday_data6_1['spread_diff'].mean()
pnl6_1_wc = pnl6_1 - 2*2*long_qty*num_contracts
if intraday_data61.empty:
pnl61 = 0
pnl61_wc = 0
else:
pnl61 = short_qty*intraday_data61['spread_diff'].mean()
pnl61_wc = pnl61 + 2*2*short_qty*num_contracts
intraday_data71 = intraday_data[(intraday_data['z6']>0.1)&(intraday_data['z5']>0.5)]
intraday_data7_1 = intraday_data[(intraday_data['z6']<-0.1)&(intraday_data['z5']<0.5)]
if intraday_data7_1.empty:
pnl7_1 = 0
pnl7_1_wc = 0
else:
pnl7_1 = long_qty*intraday_data7_1['spread_diff'].mean()
pnl7_1_wc = pnl7_1 - 2*2*long_qty*num_contracts
if intraday_data71.empty:
pnl71 = 0
pnl71_wc = 0
else:
pnl71 = short_qty*intraday_data71['spread_diff'].mean()
pnl71_wc = pnl71 + 2*2*short_qty*num_contracts
intraday_spreads['pnl1'].iloc[i] = pnl1_1 + pnl11
intraday_spreads['pnl2'].iloc[i] = pnl2_1 + pnl21
intraday_spreads['pnl5'].iloc[i] = pnl5_1 + pnl51
intraday_spreads['pnl6'].iloc[i] = pnl6_1 + pnl61
intraday_spreads['pnl7'].iloc[i] = pnl7_1 + pnl71
intraday_spreads['pnl1_wc'].iloc[i] = pnl1_1_wc + pnl11_wc
intraday_spreads['pnl2_wc'].iloc[i] = pnl2_1_wc + pnl21_wc
intraday_spreads['pnl5_wc'].iloc[i] = pnl5_1_wc + pnl51_wc
intraday_spreads['pnl6_wc'].iloc[i] = pnl6_1_wc + pnl61_wc
intraday_spreads['pnl7_wc'].iloc[i] = pnl7_1_wc + pnl71_wc
intraday_spreads.to_pickle(output_dir + '/backtest_results.pkl')
return intraday_spreads
def calc_pnl4date(**kwargs):
ticker_list = kwargs['ticker_list']
pnl_date = kwargs['pnl_date']
#print(pnl_date)
ticker_list = [x for x in ticker_list if x is not None]
ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list]
ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list]
contract_multiplier_list = [cmi.contract_multiplier[x] for x in ticker_head_list]
date_list = [exp.doubledate_shift_bus_days(double_date=pnl_date,shift_in_days=x) for x in [2, 1]]
date_list.append(pnl_date)
intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list,
date_list=date_list)
intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index]
intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date())
intraday_data['time'] = intraday_data['time_stamp'].apply(lambda x: x.time())
weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list)
spread_weights = weights_output['spread_weights']
end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list])
start_hour = max([cmi.first_trade_hour_minute[x] for x in ticker_head_list])
trade_start_hour = dt.time(9, 30, 0, 0)
num_contracts = len(ticker_list)
if 'Ag' in ticker_class_list:
start_hour1 = dt.time(0, 45, 0, 0)
end_hour1 = dt.time(7, 45, 0, 0)
selection_indx = [x for x in range(len(intraday_data.index)) if
((intraday_data['time_stamp'].iloc[x].time() < end_hour1)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or
((intraday_data['time_stamp'].iloc[x].time() < end_hour)
and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))]
else:
selection_indx = [x for x in range(len(intraday_data.index)) if
(intraday_data.index[x].to_datetime().time() < end_hour)
and(intraday_data.index[x].to_datetime().time() >= start_hour)]
intraday_data = intraday_data.iloc[selection_indx]
intraday_data['spread'] = 0
for i in range(num_contracts):
intraday_data['c' + str(i+1), 'mid_p'] = (intraday_data['c' + str(i+1)]['best_bid_p'] +
intraday_data['c' + str(i+1)]['best_ask_p'])/2
intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(i+1)]['mid_p']*spread_weights[i]
unique_settle_dates = intraday_data['settle_date'].unique()
if len(unique_settle_dates)<3:
return {'pnl_date': pnl_date, 'total_pnl': np.nan,'long_pnl':np.nan, 'short_pnl': np.nan }
final_spread_price = np.mean(intraday_data['spread'][(intraday_data['settle_date'] == unique_settle_dates[2])&(intraday_data['time']>=trade_start_hour)])
calibration_data = intraday_data[(intraday_data['settle_date'] == unique_settle_dates[0])]
backtest_data = intraday_data[(intraday_data['settle_date'] == unique_settle_dates[1])]
backtest_data = backtest_data[backtest_data['time']>=trade_start_hour]
backtest_data['spread_diff'] = contract_multiplier_list[0]*(final_spread_price-backtest_data['spread'])/spread_weights[0]
calibration_mean = calibration_data['spread'].mean()
calibration_std = calibration_data['spread'].std()
cheap_data = backtest_data[backtest_data['spread'] < calibration_mean-1*calibration_std]
expensive_data = backtest_data[backtest_data['spread'] > calibration_mean+1*calibration_std]
if cheap_data.empty:
long_pnl = 0
else:
long_pnl = cheap_data['spread_diff'].mean()-2*2*num_contracts
if expensive_data.empty:
short_pnl = 0
else:
short_pnl = -expensive_data['spread_diff'].mean()-2*2*num_contracts
return {'pnl_date': pnl_date, 'total_pnl': long_pnl+short_pnl,'long_pnl':long_pnl, 'short_pnl': short_pnl }
