def get_futures_price_preloaded(**kwargs): if 'ticker_head' in kwargs.keys(): ticker_head = kwargs['ticker_head'] else: ticker = kwargs['ticker'] ticker_head = cmi.get_contract_specs(ticker)['ticker_head'] if 'futures_data_dictionary' in kwargs.keys(): data_out = kwargs['futures_data_dictionary'][ticker_head] else: presaved_futures_data_folder = dn.get_directory_name(ext='presaved_futures_data') if os.path.isfile(presaved_futures_data_folder + '/' + ticker_head + '.pkl'): data_out = pd.read_pickle(presaved_futures_data_folder + '/' + ticker_head + '.pkl') else: data_out = pd.DataFrame() return data_out if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] if isinstance(settle_date,int): data_out = data_out[data_out['settle_date'] == cu.convert_doubledate_2datetime(settle_date)] elif isinstance(settle_date,dt.datetime): data_out = data_out[data_out['settle_date'] == settle_date] if 'settle_date_from_exclusive' in kwargs.keys(): data_out = data_out[data_out['settle_date']>cu.convert_doubledate_2datetime(kwargs['settle_date_from_exclusive'])] if 'ticker' in kwargs.keys(): data_out = data_out[data_out['ticker']==ticker] return data_out
def get_futures_price_preloaded(**kwargs): if 'ticker_head' in kwargs.keys(): ticker_head = kwargs['ticker_head'] else: ticker = kwargs['ticker'] contract_specs_output = cmi.get_contract_specs(ticker) ticker_head = contract_specs_output['ticker_head'] file_ticker = cmi.mini_contract_dictionary.get( ticker_head, ticker_head) + contract_specs_output['ticker_month_str'] + str( contract_specs_output['ticker_year']) if 'futures_data_dictionary' in kwargs.keys(): data_out = kwargs['futures_data_dictionary'][ticker_head] else: presaved_futures_data_folder = dn.get_directory_name( ext='presaved_futures_data') file_ticker_head = cmi.mini_contract_dictionary.get( ticker_head, ticker_head) if os.path.isfile(presaved_futures_data_folder + '/' + file_ticker_head + '.pkl'): data_out = pd.read_pickle(presaved_futures_data_folder + '/' + file_ticker_head + '.pkl') else: data_out = pd.DataFrame() return data_out if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] if isinstance(settle_date, int): data_out = data_out[data_out['settle_date'] == cu.convert_doubledate_2datetime(settle_date)] elif isinstance(settle_date, dt.datetime): data_out = data_out[data_out['settle_date'] == settle_date] if 'settle_date_from_exclusive' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] > cu.convert_doubledate_2datetime( kwargs['settle_date_from_exclusive'])] if 'settle_date_from' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] >= cu.convert_doubledate_2datetime( kwargs['settle_date_from'])] if 'settle_date_to' in kwargs.keys(): settle_date_to = kwargs['settle_date_to'] if isinstance(settle_date_to, int): data_out = data_out[ data_out['settle_date'] <= cu.convert_doubledate_2datetime( kwargs['settle_date_to'])] elif isinstance(settle_date_to, dt.datetime): data_out = data_out[data_out['settle_date'] <= settle_date_to] if 'ticker' in kwargs.keys(): data_out = data_out[data_out['ticker'] == file_ticker] return data_out
def get_tickers_from_agg_method_and_contracts_back(**kwargs): ticker = kwargs['ticker'] aggregation_method = kwargs['aggregation_method'] contracts_back = kwargs['contracts_back'] contact_specs_out = cmi.get_contract_specs(ticker) ref_date = 10000 * contact_specs_out[ 'ticker_year'] + 100 * contact_specs_out['ticker_month_num'] + 1 ref_datetime = cu.convert_doubledate_2datetime(ref_date) if aggregation_method == 12: cal_date_list = [ ref_datetime - relativedelta(years=x) for x in range(1, contracts_back + 1) ] elif aggregation_method == 1: cal_date_list = [ ref_datetime - relativedelta(months=x) for x in range(1, contracts_back + 1) ] ticker_list = [ contact_specs_out['ticker_head'] + cmi.full_letter_month_list[x.month - 1] + str(x.year) for x in cal_date_list ] return ticker_list
def get_symbol_list(**kwargs): quote_type = kwargs['quote_type'] datetime_from = cu.convert_doubledate_2datetime(20190317) datetime_to = dt.datetime.combine(dt.date.today(), dt.datetime.min.time()) delta = datetime_to - datetime_from # as timedelta ticker_list = [] for i in range(delta.days): day = datetime_from + dt.timedelta(days=i) ticker_frame_out = bgd.get_ticker_frame(date=int(day.strftime('%Y%m%d'))) ticker_list.extend(list(ticker_frame_out['symbol'])) ticker_list = list(set(ticker_list)) if quote_type == 'BTC': select_indx = -3 elif quote_type == 'USDT': select_indx = -4 res = [i for i in ticker_list if i[select_indx:] == quote_type] return res
def doubledate_shift_bus_days(**kwargs): if 'double_date' in kwargs.keys(): double_date = kwargs['double_date'] else: double_date = int(tm.strftime('%Y%m%d')) if 'shift_in_days' in kwargs.keys(): shift_in_days = kwargs['shift_in_days'] else: shift_in_days = 1 if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead = const.reference_tickerhead_4business_calendar bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) double_date_datetime = cu.convert_doubledate_2datetime(double_date) if shift_in_days < 0: dts_aux = pd.date_range(double_date_datetime, periods=-shift_in_days+1, freq=bday_us) dts = [x for x in dts_aux if x.to_pydatetime() != double_date_datetime] shifted_datetime = dts[-shift_in_days-1] elif shift_in_days > 0: dts_aux = pd.date_range(start=double_date_datetime-dt.timedelta(max(m.ceil(shift_in_days*7/4), shift_in_days+8)), end=double_date_datetime, freq=bday_us) dts = [x for x in dts_aux if x.to_pydatetime() != double_date_datetime] shifted_datetime = dts[-shift_in_days] return int(shifted_datetime.strftime('%Y%m%d'))
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs): ref_date = kwargs['ref_date'] aggregation_method = kwargs['aggregation_method'] contracts_back = kwargs['contracts_back'] ref_datetime = cu.convert_doubledate_2datetime(ref_date) if aggregation_method == 12: cal_date_list = [ref_datetime - relativedelta(years=x) for x in range(1, contracts_back+1)] elif aggregation_method == 1: cal_date_list = [ref_datetime - relativedelta(months=x) for x in range(1, contracts_back+1)] bday_us = CustomBusinessDay(expcalendar=exp.get_calendar_4ticker_head(const.reference_tickerhead_4business_calendar)) if 'shift_bus_days' in kwargs.keys(): shift_bus_days = kwargs['shift_bus_days'] if shift_bus_days >= 0: bus_date_list = [pd.date_range(x, periods=shift_bus_days+1, freq=bday_us)[shift_bus_days].to_datetime() for x in cal_date_list] elif shift_bus_days < 0: bus_date_list = [pd.date_range(start=x-relativedelta(days=(max(m.ceil(-shift_bus_days*7/5)+5, -shift_bus_days+5))), end=x, freq=bday_us)[shift_bus_days-1].to_datetime() for x in cal_date_list] else: bus_date_list = [pd.date_range(x, periods=1, freq=bday_us)[0].to_datetime() for x in cal_date_list] return bus_date_list
def get_days2_roll(**kwargs): ticker = kwargs['ticker'] date_to = kwargs['date_to'] contract_specs = cmf.get_contract_specs(ticker) ticker_head = contract_specs['ticker_head'] ticker_class = cmf.ticker_class[ticker_head] ticker_year = contract_specs['ticker_year'] ticker_month_num = contract_specs['ticker_month_num'] datetime_to = cu.convert_doubledate_2datetime(date_to) bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(ticker_head)) if ticker_class == 'Metal': if ticker_month_num == 1: ticker_year_roll = ticker_year-1 ticker_month_roll = 12 else: ticker_year_roll = ticker_year ticker_month_roll = ticker_month_num-1 roll_datetime = dt.date(ticker_year_roll,ticker_month_roll,25) else: expiration_datetime = get_expiration_from_db(**kwargs) roll_datetime = expiration_datetime-dt.timedelta(days=5) if roll_datetime > datetime_to.date(): dts = pd.date_range(start=datetime_to, end=roll_datetime, freq=bday_us) tr_days_2roll = len(dts)-1 else: dts = pd.date_range(start=roll_datetime, end=datetime_to, freq=bday_us) tr_days_2roll = -(len(dts)-1) return {'roll_datetime': roll_datetime, 'cal_days_2roll': (roll_datetime-datetime_to.date()).days, 'tr_days_2roll': tr_days_2roll}
def doubledate_shift_bus_days(**kwargs): if 'double_date' in kwargs.keys(): double_date = kwargs['double_date'] else: double_date = int(tm.strftime('%Y%m%d')) if 'shift_in_days' in kwargs.keys(): shift_in_days = kwargs['shift_in_days'] else: shift_in_days = 1 if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead = const.reference_tickerhead_4business_calendar bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) double_date_datetime = cu.convert_doubledate_2datetime(double_date) if shift_in_days < 0: dts_aux = pd.date_range(double_date_datetime, periods=-shift_in_days+1, freq=bday_us) dts = [x for x in dts_aux if x.to_datetime() != double_date_datetime] shifted_datetime = dts[-shift_in_days-1] elif shift_in_days > 0: dts_aux = pd.date_range(start=double_date_datetime-dt.timedelta(max(m.ceil(shift_in_days*7/4), shift_in_days+5)), end=double_date_datetime, freq=bday_us) dts = [x for x in dts_aux if x.to_datetime() != double_date_datetime] shifted_datetime = dts[-shift_in_days] return int(shifted_datetime.strftime('%Y%m%d'))
def get_binance_price_preloaded(**kwargs): interval = kwargs['interval'] ticker = kwargs['ticker'] date_from = kwargs['date_from'] date_to = kwargs['date_to'] file_name = ticker + '_1h.pkl' datetime_from = cu.convert_doubledate_2datetime(date_from) datetime_to = cu.convert_doubledate_2datetime(date_to) x = datetime_from price_frame_list = [] while x <= datetime_to: folder_name = dn.get_dated_directory_extension(folder_date=int( x.strftime('%Y%m%d')), ext='binance') dated_file_name = folder_name + '/' + file_name if os.path.isfile(dated_file_name): price_frame_list.append(pd.read_pickle(dated_file_name)) x = x + dt.timedelta(days=1) if len(price_frame_list) == 0: return pd.DataFrame() merged_data = pd.concat(price_frame_list) if len(merged_data.index) == 0: return pd.DataFrame() merged_data.set_index('openDatetime', drop=True, inplace=True) if interval.upper() != '1H': data_out = pd.DataFrame() data_out['open'] = merged_data['open'].resample('4H').first() data_out['close'] = merged_data['close'].resample('4H').last() data_out['high'] = merged_data['high'].resample('4H').max() data_out['low'] = merged_data['low'].resample('4H').min() data_out['volume'] = merged_data['volume'].resample('4H').sum() else: data_out = merged_data return data_out
def get_stock_price_preloaded(**kwargs): ticker = kwargs['ticker'] if 'data_source' in kwargs.keys(): data_source = kwargs['data_source'] else: data_source = 'iex' if 'stock_data_dictionary' in kwargs.keys(): data_out = kwargs['stock_data_dictionary'][ticker] else: if data_source == 'iex': file_dir = dn.get_directory_name(ext='iex_stock_data') else: file_dir = dn.get_directory_name(ext='stock_data') if not os.path.isfile(file_dir + '/' + ticker + '.pkl'): ssd.save_stock_data(symbol_list=[ticker], data_source=data_source) data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl') report_date = exp.doubledate_shift_bus_days() if cu.convert_doubledate_2datetime(report_date) > data_out[ 'settle_datetime'].iloc[-1].to_pydatetime(): ssd.save_stock_data(symbol_list=[ticker], data_source=data_source) data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl') if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] if isinstance(settle_date, int): data_out = data_out[data_out['settle_datetime'] == cu.convert_doubledate_2datetime(settle_date)] elif isinstance(settle_date, dt.datetime): data_out = data_out[data_out['settle_datetime'] == settle_date] if 'settle_date_from' in kwargs.keys(): data_out = data_out[ data_out['settle_datetime'] >= cu.convert_doubledate_2datetime( kwargs['settle_date_from'])] if 'settle_date_to' in kwargs.keys(): data_out = data_out[ data_out['settle_datetime'] <= cu.convert_doubledate_2datetime( kwargs['settle_date_to'])] return data_out
def get_signals_4date(**kwargs): report_date = kwargs['report_date'] date_from = cu.doubledate_shift(report_date, 365) report_datetime = cu.convert_doubledate_2datetime(report_date) fund_price_dictionary = { x: gsp.get_stock_price_preloaded(ticker=x, settle_date_from=date_from, settle_date_to=report_date) for x in symbol_list } data_current_dictionary = { x: fund_price_dictionary[x]['settle_datetime'].iloc[-1] == report_datetime for x in symbol_list } symbols2update = [] for key, value in data_current_dictionary.items(): if not value: symbols2update.append(key) if symbols2update: ssd.save_stock_data(symbol_list=symbols2update) fund_price_dictionary = { x: gsp.get_stock_price_preloaded(ticker=x, settle_date_from=date_from, settle_date_to=report_date) for x in symbol_list } performance_dictionary = {} for j in range((len(symbol_list))): price_data = fund_price_dictionary[symbol_list[j]] if price_data['settle_datetime'].iloc[-1] != report_datetime: return {'success': False, 'performance_dictionary': {}} price_data.reset_index(drop=True, inplace=True) #split_envents = price_data[price_data['split_coefficient'] != 1] #split_index_list = split_envents.index #for i in range(len(split_index_list)): # price_data['close'].iloc[:split_index_list[i]] = \ # price_data['close'].iloc[:split_index_list[i]] / price_data['split_coefficient'].iloc[split_index_list[i]] # price_data['dividend_amount'].iloc[:split_index_list[i]] = \ # price_data['dividend_amount'].iloc[:split_index_list[i]] / price_data['split_coefficient'].iloc[split_index_list[i]] performance_dictionary[symbol_list[j]] = 100 * ( price_data['close'].iloc[-1] - price_data['close'].iloc[0]) / price_data['close'].iloc[0] return {'success': True, 'performance_dictionary': performance_dictionary}
def get_rolling_curve_data(**kwargs): ticker_head = kwargs['ticker_head'] num_contracts = kwargs['num_contracts'] front_tr_dte_limit = kwargs['front_tr_dte_limit'] date_from = kwargs['date_from'] date_to = kwargs['date_to'] date_from_datetime = cu.convert_doubledate_2datetime(date_from) date_to_datetime = cu.convert_doubledate_2datetime(date_to) panel_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head) panel_data = panel_data.loc[(panel_data['settle_date'] >= date_from_datetime) & (panel_data['settle_date'] <= date_to_datetime) & (panel_data['tr_dte'] >= front_tr_dte_limit)] if 'month_separation' in kwargs.keys(): month_separation = kwargs['month_separation'] elif ticker_head == 'ED': month_separation = 3 else: month_separation = 1 if month_separation != 1: panel_data = panel_data[panel_data['ticker_month'] % month_separation == 0] panel_data = panel_data[np.isfinite(panel_data['close_price'])] sorted_data = panel_data.sort(['settle_date', 'tr_dte'], ascending=[True, True]) filtered_data = sorted_data.groupby('settle_date').filter(lambda x:len(x)>=num_contracts) filtered_data2 = filtered_data.groupby('settle_date').filter(lambda x: all([cmi.get_month_seperation_from_cont_indx(x['cont_indx'].values[i], x['cont_indx'].values[i+1]) ==- month_separation for i in range(num_contracts-1)])) grouped = filtered_data2.groupby('settle_date') rolling_data_list = [] for i in range(num_contracts): rolling_data_list.append(grouped.nth(i)) return rolling_data_list
def get_rolling_curve_data(**kwargs): ticker_head = kwargs['ticker_head'] num_contracts = kwargs['num_contracts'] front_tr_dte_limit = kwargs['front_tr_dte_limit'] date_from = kwargs['date_from'] date_to = kwargs['date_to'] date_from_datetime = cu.convert_doubledate_2datetime(date_from) date_to_datetime = cu.convert_doubledate_2datetime(date_to) panel_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head) panel_data = panel_data.loc[(panel_data['settle_date'] >= date_from_datetime) & (panel_data['settle_date'] <= date_to_datetime) & (panel_data['tr_dte'] >= front_tr_dte_limit)] if 'month_separation' in kwargs.keys(): month_separation = kwargs['month_separation'] elif ticker_head == 'ED': month_separation = 3 else: month_separation = 1 if month_separation != 1: panel_data = panel_data[panel_data['ticker_month'] % month_separation == 0] panel_data = panel_data[np.isfinite(panel_data['close_price'])] sorted_data = panel_data.sort_values(['settle_date', 'tr_dte'], ascending=[True, True]) filtered_data = sorted_data.groupby('settle_date').filter(lambda x:len(x)>=num_contracts) filtered_data2 = filtered_data.groupby('settle_date').filter(lambda x: all([cmi.get_month_seperation_from_cont_indx(x['cont_indx'].values[i], x['cont_indx'].values[i+1]) ==- month_separation for i in range(num_contracts-1)])) grouped = filtered_data2.groupby('settle_date') rolling_data_list = [] for i in range(num_contracts): rolling_data_list.append(grouped.nth(i)) return rolling_data_list
def get_butterfly_panel_plot(**kwargs): report_date = kwargs['report_date'] id = kwargs['id'] bf_output = fb.generate_futures_butterfly_sheet_4date(date_to=report_date) butterflies = bf_output['butterflies'] contract_list = [butterflies['ticker1'][id], butterflies['ticker2'][id], butterflies['ticker3'][id]] tr_dte_list = [butterflies['trDte1'][id], butterflies['trDte2'][id], butterflies['trDte3'][id]] if 'aggregation_method' in kwargs.keys(): aggregation_method = kwargs['aggregation_method'] else: aggregation_method = butterflies['agg'][id] if 'contracts_back' in kwargs.keys(): contracts_back = kwargs['contracts_back'] else: contracts_back = butterflies['cBack'][id] post_report_date = exp.doubledate_shift_bus_days(double_date=report_date,shift_in_days=-20) bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=contract_list, tr_dte_list=tr_dte_list, aggregation_method=aggregation_method, contracts_back=contracts_back, date_to=post_report_date, contract_multiplier=butterflies['multiplier'][id], use_last_as_current=True) aligned_data = bf_signals_output['aligned_output']['aligned_data'] new_index = list(range(len(aligned_data.index))) contract_change_indx = (aligned_data['c1']['ticker_year']-aligned_data['c1']['ticker_year'].shift(1)!=0).values front_contract_year = aligned_data['c1']['ticker_year'] % 10 contract_change_indx[0] = False report_datetime = cu.convert_doubledate_2datetime(report_date) x_index = [x for x in new_index if aligned_data['settle_date'][x] == report_datetime][0] x_tick_locations = [x for x in new_index if contract_change_indx[x]] x_tick_locations.append(x_index) x_tick_values = [cmi.letter_month_string[aligned_data['c1']['ticker_month'].values[x]-1]+ str(front_contract_year.values[x]) for x in new_index if contract_change_indx[x]] x_tick_values.append('X') plt.figure(figsize=(16, 7)) plt.plot(aligned_data['residuals']) plt.xticks(x_tick_locations,x_tick_values) plt.grid() plt.title('Contracts: ' + str(contract_list) + ', weight2: ' + str(bf_signals_output['second_spread_weight_1'].round(2))) plt.show() return bf_signals_output
def get_butterfly_panel_plot(**kwargs): report_date = kwargs['report_date'] id = kwargs['id'] bf_output = fb.generate_futures_butterfly_sheet_4date(date_to=report_date) butterflies = bf_output['butterflies'] contract_list = [butterflies['ticker1'][id], butterflies['ticker2'][id], butterflies['ticker3'][id]] tr_dte_list = [butterflies['trDte1'][id], butterflies['trDte2'][id], butterflies['trDte3'][id]] if 'aggregation_method' in kwargs.keys(): aggregation_method = kwargs['aggregation_method'] else: aggregation_method = butterflies['agg'][id] if 'contracts_back' in kwargs.keys(): contracts_back = kwargs['contracts_back'] else: contracts_back = butterflies['cBack'][id] post_report_date = exp.doubledate_shift_bus_days(double_date=report_date,shift_in_days=-20) bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=contract_list, tr_dte_list=tr_dte_list, aggregation_method=aggregation_method, contracts_back=contracts_back, date_to=post_report_date, contract_multiplier=butterflies['multiplier'][id], use_last_as_current=True) aligned_data = bf_signals_output['aligned_output']['aligned_data'] new_index = list(range(len(aligned_data.index))) contract_change_indx = (aligned_data['c1']['ticker_year']-aligned_data['c1']['ticker_year'].shift(1)!=0).values front_contract_year = aligned_data['c1']['ticker_year'] % 10 contract_change_indx[0] = False report_datetime = cu.convert_doubledate_2datetime(report_date) x_index = [x for x in new_index if aligned_data['settle_date'][x] == report_datetime][0] x_tick_locations = [x for x in new_index if contract_change_indx[x]] x_tick_locations.append(x_index) x_tick_values = [cmi.letter_month_string[aligned_data['c1']['ticker_month'].values[x]-1]+ str(front_contract_year.values[x]) for x in new_index if contract_change_indx[x]] x_tick_values.append('X') plt.figure(figsize=(16, 7)) plt.plot(aligned_data['residuals']) plt.xticks(x_tick_locations,x_tick_values) plt.grid() plt.title('Contracts: ' + str(contract_list) + ', weight2: ' + str(bf_signals_output['second_spread_weight_1'].round(2))) plt.show() return bf_signals_output
def get_cot_data(**kwargs): ticker_head = kwargs['ticker_head'] if os.path.isfile(presaved_cot_data_folder + '/' + ticker_head + '.pkl'): data_out = pd.read_pickle(presaved_cot_data_folder + '/' + ticker_head + '.pkl') if 'date_from' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] >= cu.convert_doubledate_2datetime( kwargs['date_from'])] if 'date_to' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] <= cu.convert_doubledate_2datetime(kwargs['date_to']) - dt.timedelta(days=3)] else: data_out = pd.DataFrame() return data_out
def get_contract_summary_stats(**kwargs): ticker = kwargs['ticker'] date_to = kwargs['date_to'] data_out = gfp.get_futures_price_preloaded(ticker=ticker) datetime_to = cu.convert_doubledate_2datetime(date_to) data_out = data_out[data_out['settle_date'] <= datetime_to] data_out['close_price_daily_diff'] = data_out['close_price'] - data_out['close_price'].shift(1) daily_noise = np.std(data_out['close_price_daily_diff'].iloc[-60:]) average_volume = np.mean(data_out['volume'].iloc[-20:]) return {'daily_noise': daily_noise, 'average_volume':average_volume}
def get_results_4date(**kwargs): date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) output_dir = ts.create_strategy_output_dir( strategy_class='futures_directional', report_date=date_to) if os.path.isfile(output_dir + '/summary.pkl'): directionals = pd.read_pickle(output_dir + '/summary.pkl') corr_matrix = pd.read_pickle(output_dir + '/corr.pkl') return { 'directionals': directionals, 'corr_matrix': corr_matrix, 'success': True } ticker_head_list = list( set(cmi.cme_futures_tickerhead_list + cmi.futures_butterfly_strategy_tickerhead_list)) results_output = [ get_results_4tickerhead(ticker_head=x, date_to=date_to) for x in ticker_head_list ] rolling_data_list = [x.pop('rolling_data') for x in results_output] directionals = pd.DataFrame(results_output) directionals['ticker_head'] = ticker_head_list directionals['weekday'] = datetime_to.weekday() directionals.to_pickle(output_dir + '/summary.pkl') aux_frame_list = [] for i in range(len(ticker_head_list)): rolling_data = rolling_data_list[i].iloc[-20:] if not rolling_data.empty: aux_frame = pd.DataFrame(index=rolling_data['settle_date']) aux_frame[ticker_head_list[i]] = rolling_data['change_1'].values aux_frame_list.append(aux_frame) if len(aux_frame_list) > 1: combined_frame = pd.concat(aux_frame_list, axis=1, join='inner') corr_matrix = combined_frame.corr() else: corr_matrix = pd.DataFrame() corr_matrix.to_pickle(output_dir + '/corr.pkl') return { 'directionals': directionals, 'corr_matrix': corr_matrix, 'success': True }
def get_futures_days2_expiration(expiration_input): ticker = expiration_input['ticker'] date_to = expiration_input['date_to'] contract_specs_output = cmf.get_contract_specs(ticker) bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head'])) expiration_date = get_futures_expiration(ticker) dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_to), end=expiration_date, freq=bday_us) return len(dts)-1
def get_futures_days2_expiration(expiration_input): ticker = expiration_input['ticker'] date_to = expiration_input['date_to'] contract_specs_output = cmf.get_contract_specs(ticker) bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head'])) expiration_date = get_futures_expiration(ticker) dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_to), end=expiration_date, freq=bday_us) return len(dts)-1
def save_price_data(**kwargs): if 'client' in kwargs.keys(): client = kwargs['client'] else: client = btu.get_binance_client() interval = kwargs['interval'] ticker = kwargs['ticker'] date_from = kwargs['date_from'] date_to = kwargs['date_to'] file_name = ticker + '_' + interval + '.pkl' datetime_from = cu.convert_doubledate_2datetime(date_from) datetime_to = cu.convert_doubledate_2datetime(date_to) x = datetime_from while x <= datetime_to: xplus = x + dt.timedelta(days=1) folder_name = dn.get_dated_directory_extension(folder_date=int( x.strftime('%Y%m%d')), ext='binance') dated_file_name = folder_name + '/' + file_name if os.path.isfile(dated_file_name): price_frame = pd.read_pickle(dated_file_name) print(len(price_frame.index)) else: price_frame = gbp.get_klines(ticker=ticker, interval=interval, start_str=x.strftime('%m/%d/%y'), end_str=xplus.strftime('%m/%d/%y'), client=client) price_frame = price_frame[price_frame['openDate'] == x.date()] price_frame.to_pickle(dated_file_name) tm.sleep(0.5) x = xplus
def get_backtesting_dates(**kwargs): date_to = kwargs['date_to'] years_back = kwargs['years_back'] if 'day_of_week' in kwargs.keys(): day_of_week = kwargs['day_of_week'] else: day_of_week = 2 date_from = cu.doubledate_shift(date_to, years_back*365) trading_calendar = exp.get_calendar_4ticker_head('CL') bday_us = CustomBusinessDay(calendar=trading_calendar) dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from), end=cu.convert_doubledate_2datetime(date_to), freq=bday_us) dts = dts[dts.dayofweek==day_of_week] return {'date_time_dates': dts, 'double_dates': [int(x.strftime('%Y%m%d')) for x in dts]}
def download_coinbase_data_4ticker(**kwargs): date_to = kwargs["date_to"] num_days_back = kwargs["num_days_back"] datetime_to = cu.convert_doubledate_2datetime(date_to) date_list = [ int((datetime_to - dt.timedelta(days=x)).strftime('%Y%m%d')) for x in range(0, num_days_back) ] [download_coinbase_data_4date(**kwargs, utc_date=x) for x in date_list]
def get_summary(**kwargs): symbol1 = kwargs['symbol1'] symbol2 = kwargs['symbol2'] report_date = kwargs['report_date'] if 'get_diagnosticQ' in kwargs.keys(): get_diagnosticQ = kwargs['get_diagnosticQ'] else: get_diagnosticQ = False report_datetime = cu.convert_doubledate_2datetime(report_date) data1 = gsp.get_stock_price_preloaded(ticker=symbol1, data_source='iex', settle_date_to = report_date) data2 = gsp.get_stock_price_preloaded(ticker=symbol2, data_source='iex', settle_date_to = report_date) merged_data = pd.merge(data1[['close','settle_datetime']], data2[['close','settle_datetime']], how='inner', on='settle_datetime') split = int(len(merged_data) * .4) if split<200 or report_datetime!=merged_data['settle_datetime'].iloc[-1]: return {'price1': np.nan,'price2': np.nan, 'p_value_2': np.nan,'p_value_1': np.nan, 'beta_1': np.nan, 'beta_2': np.nan, 'corr': np.nan, 'cagr1': np.nan, 'cagr2': np.nan, 'kalpha': np.nan, 'kbeta': np.nan, 'meanSpread': np.nan, 'stdSpread': np.nan, 'zScore': np.nan} training_data = merged_data[:split] test_data = merged_data[split:] cointegration_output_2 = sm.tsa.stattools.coint(training_data['close_x'], training_data['close_y']) cointegration_output_1 = sm.tsa.stattools.coint(test_data['close_x'], test_data['close_y']) regress_output_1 = ss.get_regression_results({'y': test_data['close_y'].values, 'x': test_data['close_x'].values}) regress_output_2 = ss.get_regression_results({'y': training_data['close_y'].values, 'x': training_data['close_x'].values}) regress_output_3 = ss.get_regression_results({'y': test_data['close_y'].diff().values, 'x': test_data['close_x'].diff().values}) merged_data.set_index('settle_datetime', drop=True, inplace=True) backtest_output_1 = backtest(merged_data[split:], 'close_x', 'close_y') backtest_output_2 = backtest(merged_data[:split], 'close_x', 'close_y') if get_diagnosticQ: return {'backtest_output': backtest_output_1, 'cagr2': backtest_output_2['cagr']} else: return {'price1': merged_data['close_x'].iloc[-1],'price2': merged_data['close_y'].iloc[-1], 'p_value_2': cointegration_output_2[1],'p_value_1': cointegration_output_1[1], 'beta_1': regress_output_1['beta'], 'beta_2': regress_output_2['beta'], 'corr': np.sqrt(regress_output_3['rsquared']/100), 'cagr1': backtest_output_1['cagr'], 'cagr2': backtest_output_2['cagr'], 'kalpha': backtest_output_1['kalpha'], 'kbeta': backtest_output_1['kbeta'], 'meanSpread': backtest_output_1['meanSpread'], 'stdSpread': backtest_output_1['stdSpread'], 'zScore': backtest_output_1['zScore']}
def get_bus_day_list(**kwargs): if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead =const.reference_tickerhead_4business_calendar if 'date_from' in kwargs.keys(): datetime_from = cu.convert_doubledate_2datetime(kwargs['date_from']) if 'datetime_from' in kwargs.keys(): datetime_from = kwargs['datetime_from'] if 'date_to' in kwargs.keys(): datetime_to = cu.convert_doubledate_2datetime(kwargs['date_to']) if 'datetime_to' in kwargs.keys(): datetime_to = kwargs['datetime_to'] bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) date_index = pd.date_range(start=datetime_from, end=datetime_to, freq=bday_us) return [int(x.to_pydatetime().strftime('%Y%m%d')) for x in date_index]
def get_bus_day_list(**kwargs): if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead =const.reference_tickerhead_4business_calendar if 'date_from' in kwargs.keys(): datetime_from = cu.convert_doubledate_2datetime(kwargs['date_from']) if 'datetime_from' in kwargs.keys(): datetime_from = kwargs['datetime_from'] if 'date_to' in kwargs.keys(): datetime_to = cu.convert_doubledate_2datetime(kwargs['date_to']) if 'datetime_to' in kwargs.keys(): datetime_to = kwargs['datetime_to'] bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) date_index = pd.date_range(start=datetime_from, end=datetime_to, freq=bday_us) return [int(x.to_datetime().strftime('%Y%m%d')) for x in date_index]
def load_trades_2strategy(**kwargs): trade_frame = kwargs['trade_frame'] con = msu.get_my_sql_connection(**kwargs) trade_frame['strategy_id'] = [ get_strategy_id_from_alias(alias=trade_frame['alias'][x], con=con) for x in range(len(trade_frame.index)) ] trade_frame['strike_price'] = trade_frame['strike_price'].astype('float64') now_time = dt.datetime.now() now_date = now_time.date() if 'trade_date' in kwargs.keys(): trade_date = cu.convert_doubledate_2datetime(kwargs['trade_date']) else: trade_date = now_date column_str = "ticker, option_type, strike_price, strategy_id, trade_price, trade_quantity, trade_date, instrument, real_tradeQ, created_date, last_updated_date" insert_str = ("%s, " * 11)[:-2] final_str = "INSERT INTO trades (%s) VALUES (%s)" % (column_str, insert_str) column_names = trade_frame.columns.tolist() ticker_indx = column_names.index('ticker') option_type_indx = column_names.index('option_type') strike_price_indx = column_names.index('strike_price') trade_price_indx = column_names.index('trade_price') trade_quantity_indx = column_names.index('trade_quantity') instrument_indx = column_names.index('instrument') real_tradeQ_indx = column_names.index('real_tradeQ') strategy_id_indx = column_names.index('strategy_id') tuples = [ tuple([ x[ticker_indx], None if pd.isnull(x[option_type_indx]) else x[option_type_indx], None if pd.isnull(x[strike_price_indx]) else x[strike_price_indx], x[strategy_id_indx], x[trade_price_indx], x[trade_quantity_indx], trade_date, x[instrument_indx], x[real_tradeQ_indx], now_time, now_time ]) for x in trade_frame.values ] msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con) if 'con' not in kwargs.keys(): con.close()
def get_days2_expiration(**kwargs): date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) expiration_datetime = get_expiration_from_db(**kwargs) contract_specs_output = cmf.get_contract_specs(kwargs['ticker']) bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head'])) dts = pd.date_range(start=datetime_to, end=expiration_datetime, freq=bday_us) return {'expiration_datetime': expiration_datetime, 'cal_dte': (expiration_datetime-datetime_to.date()).days, 'tr_dte': len(dts)-1}
def get_daily_price_data4ticker(**kwargs): date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) try: data_out = pd.read_pickle( dna.get_directory_name(ext='binance') + '/daily/' + kwargs['ticker'] + '.pkl') except: return pd.DataFrame() #data_out['pydate'] = [x.to_pydatetime().date() for x in data_out['openDatetime']] return data_out[data_out['openDate'] <= datetime_to.date()]
def get_days2_expiration(**kwargs): date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) expiration_datetime = get_expiration_from_db(**kwargs) contract_specs_output = cmf.get_contract_specs(kwargs['ticker']) bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(contract_specs_output['ticker_head'])) dts = pd.date_range(start=datetime_to, end=expiration_datetime, freq=bday_us) return {'expiration_datetime': expiration_datetime, 'cal_dte': (expiration_datetime-datetime_to.date()).days, 'tr_dte': len(dts)-1}
def get_backtesting_dates(**kwargs): date_to = kwargs['date_to'] years_back = kwargs['years_back'] if 'day_of_week' in kwargs.keys(): day_of_week = kwargs['day_of_week'] else: day_of_week = 2 date_from = cu.doubledate_shift(date_to, years_back * 365) trading_calendar = exp.get_calendar_4ticker_head('CL') bday_us = CustomBusinessDay(calendar=trading_calendar) dts = pd.date_range(start=cu.convert_doubledate_2datetime(date_from), end=cu.convert_doubledate_2datetime(date_to), freq=bday_us) dts = dts[dts.dayofweek == day_of_week] return { 'date_time_dates': dts, 'double_dates': [int(x.strftime('%Y%m%d')) for x in dts] }
def is_business_day(**kwargs): double_date = kwargs['double_date'] if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead = const.reference_tickerhead_4business_calendar bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) double_date_datetime = cu.convert_doubledate_2datetime(double_date) dts_aux = pd.date_range(double_date_datetime, periods=1, freq=bday_us) return double_date_datetime in dts_aux
def is_business_day(**kwargs): double_date = kwargs['double_date'] if 'reference_tickerhead' in kwargs.keys(): reference_tickerhead = kwargs['reference_tickerhead'] else: reference_tickerhead = const.reference_tickerhead_4business_calendar bday_us = CustomBusinessDay(calendar=get_calendar_4ticker_head(reference_tickerhead)) double_date_datetime = cu.convert_doubledate_2datetime(double_date) dts_aux = pd.date_range(double_date_datetime, periods=1, freq=bday_us) return double_date_datetime in dts_aux
def calc_realized_vol_4futures_ticker(**kwargs): settle_date = kwargs.pop('settle_date') num_obs = kwargs.pop('num_obs', 20) futures_price_output = gfp.get_futures_price_preloaded(**kwargs) settle_datetime = cu.convert_doubledate_2datetime(settle_date) futures_price_selected = futures_price_output[futures_price_output['settle_date'] <= settle_datetime] logreturns = np.log(futures_price_selected['close_price'][-(num_obs+1):]/ futures_price_selected['close_price'][-(num_obs+1):].shift(1)) return 100*np.sqrt(252*np.mean(np.square(logreturns)))
def calc_realized_vol_4futures_ticker(**kwargs): settle_date = kwargs.pop('settle_date') num_obs = kwargs.pop('num_obs', 20) futures_price_output = gfp.get_futures_price_preloaded(**kwargs) settle_datetime = cu.convert_doubledate_2datetime(settle_date) futures_price_selected = futures_price_output[futures_price_output['settle_date'] <= settle_datetime] logreturns = np.log(futures_price_selected['close_price'][-(num_obs+1):]/ futures_price_selected['close_price'][-(num_obs+1):].shift(1)) return 100*np.sqrt(252*np.mean(np.square(logreturns)))
def get_bus_dates_from_agg_method_and_contracts_back(**kwargs): ref_date = kwargs['ref_date'] aggregation_method = kwargs['aggregation_method'] contracts_back = kwargs['contracts_back'] ref_datetime = cu.convert_doubledate_2datetime(ref_date) if aggregation_method == 12: cal_date_list = [ ref_datetime - relativedelta(years=x) for x in range(1, contracts_back + 1) ] elif aggregation_method == 1: cal_date_list = [ ref_datetime - relativedelta(months=x) for x in range(1, contracts_back + 1) ] bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head( const.reference_tickerhead_4business_calendar)) if 'shift_bus_days' in kwargs.keys(): shift_bus_days = kwargs['shift_bus_days'] if shift_bus_days >= 0: bus_date_list = [ pd.date_range(x, periods=shift_bus_days + 1, freq=bday_us)[shift_bus_days].to_pydatetime() for x in cal_date_list ] elif shift_bus_days < 0: bus_date_list = [ pd.date_range(start=x - relativedelta(days=(max( m.ceil(-shift_bus_days * 7 / 5) + 5, -shift_bus_days + 5))), end=x, freq=bday_us)[shift_bus_days - 1].to_pydatetime() for x in cal_date_list ] else: bus_date_list = [ pd.date_range(x, periods=1, freq=bday_us)[0].to_pydatetime() for x in cal_date_list ] return bus_date_list
def download_coinbase_data_4date(**kwargs): utc_date = kwargs['utc_date'] print(utc_date) candlestick_minutes = kwargs['candlestick_minutes'] ticker = kwargs['ticker'] coinbase_data_dir = dn.get_dated_directory_extension(folder_date=utc_date, ext='coinbase_data') file_name = coinbase_data_dir + '/' + ticker + '_' + str( candlestick_minutes) + '.pkl' if os.path.isfile(file_name): return pd.read_pickle(file_name) if 'coin_client' in kwargs.keys(): coin_client = kwargs['coin_client'] else: coin_client = coin_util.get_coinbase_client() start_date = cu.convert_doubledate_2datetime(utc_date) start_date = start_date.replace(hour=0, minute=0) end_date = start_date + dt.timedelta(days=1) end_date = end_date.replace(hour=0, minute=5) date_raw = coin_client.get_product_historic_rates( ticker, granularity=candlestick_minutes * 60, start=start_date, end=end_date) time.sleep(0.5) frame_out = pd.DataFrame( date_raw, columns=['time', 'low', 'high', 'open', 'close', 'volume']) frame_out['time'] = [ dt.datetime.utcfromtimestamp(x).replace(tzinfo=pytz.utc) for x in frame_out['time'] ] frame_out.sort_values(by='time', ascending=True, inplace=True) frame_out.reset_index(drop=True, inplace=True) frame_out.to_pickle(file_name) return frame_out
def get_spread_carry_4tickerhead(**kwargs): ticker_head = kwargs['ticker_head'] report_date = kwargs['report_date'] if 'min_tr_dte' in kwargs.keys(): min_tr_dte = kwargs['min_tr_dte'] else: min_tr_dte = 15 if 'futures_data_dictionary' in kwargs.keys(): futures_data_dictionary = kwargs['futures_data_dictionary'] else: futures_data_dictionary = { ticker_head: gfp.get_futures_price_preloaded(ticker_head=ticker_head) } if 'datetime5_years_ago' in kwargs.keys(): datetime5_years_ago = kwargs['datetime5_years_ago'] else: date5_years_ago = cu.doubledate_shift(report_date, 5 * 365) datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago) daily_data = gfp.get_futures_price_preloaded( ticker_head=ticker_head, settle_date=report_date, futures_data_dictionary=futures_data_dictionary) daily_data = daily_data[(daily_data['tr_dte'] >= min_tr_dte) & ( daily_data['tr_dte'] <= max_tr_dte_limits[ticker_head])] if len(daily_data.index) > 1: carry_signals = fs.get_futures_spread_carry_signals( ticker_list=daily_data['ticker'].values, tr_dte_list=daily_data['tr_dte'].values, futures_data_dictionary=futures_data_dictionary, datetime5_years_ago=datetime5_years_ago, date_to=report_date) return {'success': True, 'carry_signals': carry_signals} else: return {'success': False, 'carry_signals': pd.DataFrame()}
def get_altcoin_demand(**kwargs): date_to = kwargs['date_to'] num_days_back = kwargs['num_days_back'] datetime_to = cu.convert_doubledate_2datetime(date_to) datetime_list = [ cu.get_datetime_shift(reference_date=datetime_to, shift_in_days=x) for x in range(num_days_back) ] datetime_list.reverse() date_list = [int(x.strftime('%Y%m%d')) for x in datetime_list] demand_list = [] for i in range(len(date_list)): ticker_frame_out = gd.get_ticker_frame(date=date_list[i]) btc_frame = ticker_frame_out.loc[[ x[-3:] == 'BTC' for x in ticker_frame_out['symbol'] ]] btc_frame['quoteVolume'] = pd.to_numeric(btc_frame['quoteVolume']) btc_frame['priceChangePercent'] = pd.to_numeric( btc_frame['priceChangePercent']) btc_frame.sort_values(['quoteVolume'], inplace=True, ascending=False) btc_frame = btc_frame.iloc[:50] btc_frame['averageVolume'] = [ get_average_volume(ticker=x, date_to=date_list[i]) for x in btc_frame['symbol'] ] btc_frame.sort_values(['averageVolume'], inplace=True, ascending=False) btc_frame = btc_frame.iloc[:10] demand_list.append(np.mean(btc_frame['priceChangePercent'])) data_out = pd.DataFrame() data_out['date'] = date_list data_out['datetime'] = datetime_list data_out['demand'] = demand_list data_out['cumulative'] = data_out['demand'].cumsum() data_out['pydate'] = [x.date() for x in data_out['datetime']] return data_out
def get_tickers_from_agg_method_and_contracts_back(**kwargs): ticker = kwargs['ticker'] aggregation_method = kwargs['aggregation_method'] contracts_back = kwargs['contracts_back'] contact_specs_out = cmi.get_contract_specs(ticker) ref_date = 10000*contact_specs_out['ticker_year']+100*contact_specs_out['ticker_month_num']+1 ref_datetime = cu.convert_doubledate_2datetime(ref_date) if aggregation_method == 12: cal_date_list = [ref_datetime - relativedelta(years=x) for x in range(1, contracts_back+1)] elif aggregation_method == 1: cal_date_list = [ref_datetime - relativedelta(months=x) for x in range(1, contracts_back+1)] ticker_list = [contact_specs_out['ticker_head'] + cmi.full_letter_month_list[x.month-1] + str(x.year) for x in cal_date_list] return ticker_list
def get_intraday_breakout_chart(**kwargs): signal_out = ifs.get_intraday_trend_signals(ticker=kwargs['ticker'],date_to=kwargs['trade_date']) intraday_data = signal_out['intraday_data'] trade_datetime = cu.convert_doubledate_2datetime(kwargs['trade_date']) intraday_data = intraday_data[intraday_data['time_stamp']>=trade_datetime] plt.figure(figsize=(16, 7)) plt.plot(intraday_data['time_stamp'],intraday_data['mid_p'],color='k') plt.plot(intraday_data['time_stamp'],intraday_data['ewma25'],color='b') plt.plot(intraday_data['time_stamp'],intraday_data['ewma100'],color='g') plt.axvline(dt.datetime.combine(trade_datetime,dt.time(8,30,0,0)),color='r') plt.axvline(dt.datetime.combine(trade_datetime,dt.time(9,0,0,0)),color='r') #plt.axvline([x for x in intraday_data['time_stamp'] if 100*x.hour+x.minute == 830][0],color='r') #plt.axvline([x for x in intraday_data.index if 100*x.hour+x.minute == 900][0],color='r') plt.grid() plt.show()
def load_trades_2strategy(**kwargs): trade_frame = kwargs['trade_frame'] con = msu.get_my_sql_connection(**kwargs) trade_frame['strategy_id'] = [get_strategy_id_from_alias(alias=trade_frame['alias'][x],con=con) for x in range(len(trade_frame.index))] trade_frame['strike_price'] = trade_frame['strike_price'].astype('float64') now_time = dt.datetime.now() now_date = now_time.date() if 'trade_date' in kwargs.keys(): trade_date = cu.convert_doubledate_2datetime(kwargs['trade_date']) else: trade_date = now_date column_str = "ticker, option_type, strike_price, strategy_id, trade_price, trade_quantity, trade_date, instrument, real_tradeQ, created_date, last_updated_date" insert_str = ("%s, " * 11)[:-2] final_str = "INSERT INTO trades (%s) VALUES (%s)" % (column_str, insert_str) column_names = trade_frame.columns.tolist() ticker_indx = column_names.index('ticker') option_type_indx = column_names.index('option_type') strike_price_indx = column_names.index('strike_price') trade_price_indx = column_names.index('trade_price') trade_quantity_indx = column_names.index('trade_quantity') instrument_indx = column_names.index('instrument') real_tradeQ_indx = column_names.index('real_tradeQ') strategy_id_indx = column_names.index('strategy_id') tuples = [tuple([x[ticker_indx],x[option_type_indx], None if np.isnan(x[strike_price_indx]) else x[strike_price_indx], x[strategy_id_indx],x[trade_price_indx], x[trade_quantity_indx], trade_date,x[instrument_indx], x[real_tradeQ_indx],now_time,now_time]) for x in trade_frame.values] msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con) if 'con' not in kwargs.keys(): con.close()
def generate_db_strategy_from_alias(**kwargs): alias = kwargs['alias'] description_string = kwargs['description_string'] con = msu.get_my_sql_connection(**kwargs) cur = con.cursor() for i in range(1, 50): if i > 1: alias_modified = alias + '_' + str(i) else: alias_modified = alias strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con) if not strategy_id: break now_date = dt.datetime.now().date() column_str = "alias, open_date, close_date, created_date, last_updated_date, description_string" insert_str = ("%s, " * 6)[:-2] tuple_to_load = (alias_modified, now_date, cu.convert_doubledate_2datetime(30000101), now_date, now_date, description_string) final_str = "INSERT INTO strategy (%s) VALUES (%s)" % (column_str, insert_str) cur.execute(final_str, tuple_to_load) con.commit() strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con) if 'con' not in kwargs.keys(): con.close() return {'alias': alias_modified, 'strategy_id': strategy_id}
def get_net_position_4strategy_alias(**kwargs): alias = kwargs['alias'] if 'as_of_date' in kwargs.keys(): as_of_date = kwargs['as_of_date'] else: as_of_date = exp.doubledate_shift_bus_days() con = msu.get_my_sql_connection(**kwargs) trades_frame = get_trades_4strategy_alias(alias=alias, con=con) as_of_datetime = cu.convert_doubledate_2datetime(as_of_date) trades_frame = trades_frame[trades_frame['trade_date'] <= as_of_datetime] trades_frame['full_ticker'] = [ trades_frame['ticker'].iloc[x] if trades_frame['instrument'].iloc[x] in ['F', 'S'] else trades_frame['ticker'].iloc[x] + '_' + trades_frame['option_type'].iloc[x] + str(trades_frame['strike_price'].iloc[x]) for x in range(len(trades_frame.index)) ] grouped = trades_frame.groupby('full_ticker') net_position = pd.DataFrame() net_position['ticker'] = (grouped['ticker'].first()).values net_position['option_type'] = (grouped['option_type'].first()).values net_position['strike_price'] = (grouped['strike_price'].first()).values net_position['instrument'] = (grouped['instrument'].first()).values net_position['qty'] = (grouped['trade_quantity'].sum()).values net_position['qty'] = net_position['qty'].round(2) if 'con' not in kwargs.keys(): con.close() return net_position[net_position['qty'] != 0]
def get_spread_carry_4tickerhead(**kwargs): ticker_head = kwargs['ticker_head'] report_date = kwargs['report_date'] if 'min_tr_dte' in kwargs.keys(): min_tr_dte = kwargs['min_tr_dte'] else: min_tr_dte = 10 if 'futures_data_dictionary' in kwargs.keys(): futures_data_dictionary = kwargs['futures_data_dictionary'] else: futures_data_dictionary = {ticker_head: gfp.get_futures_price_preloaded(ticker_head=ticker_head)} if 'datetime5_years_ago' in kwargs.keys(): datetime5_years_ago = kwargs['datetime5_years_ago'] else: date5_years_ago = cu.doubledate_shift(report_date,5*365) datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago) daily_data = gfp.get_futures_price_preloaded(ticker_head=ticker_head, settle_date=report_date, futures_data_dictionary=futures_data_dictionary) daily_data = daily_data[(daily_data['tr_dte'] >= min_tr_dte) & (daily_data['tr_dte'] <= max_tr_dte_limits[ticker_head])] if len(daily_data.index) > 1: carry_signals = fs.get_futures_spread_carry_signals(ticker_list=daily_data['ticker'].values, tr_dte_list=daily_data['tr_dte'].values, futures_data_dictionary=futures_data_dictionary, datetime5_years_ago=datetime5_years_ago, date_to=report_date) return {'success': True, 'carry_signals': carry_signals} else: return {'success': False, 'carry_signals': pd.DataFrame()}
def generate_db_strategy_from_alias(**kwargs): alias = kwargs['alias'] description_string = kwargs['description_string'] con = msu.get_my_sql_connection(**kwargs) cur = con.cursor() for i in range(1, 10): if i > 1: alias_modified = alias + '_' + str(i) else: alias_modified = alias strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con) if not strategy_id: break now_date = dt.datetime.now().date() column_str = "alias, open_date, close_date, created_date, last_updated_date, description_string" insert_str = ("%s, " * 6)[:-2] tuple_to_load = (alias_modified, now_date, cu.convert_doubledate_2datetime(30000101), now_date, now_date, description_string) final_str = "INSERT INTO strategy (%s) VALUES (%s)" % (column_str, insert_str) cur.execute(final_str,tuple_to_load) con.commit() strategy_id = get_strategy_id_from_alias(alias=alias_modified, con=con) if 'con' not in kwargs.keys(): con.close() return strategy_id
def get_net_position_4strategy_alias(**kwargs): alias = kwargs['alias'] if 'as_of_date' in kwargs.keys(): as_of_date = kwargs['as_of_date'] else: as_of_date = exp.doubledate_shift_bus_days() con = msu.get_my_sql_connection(**kwargs) trades_frame = get_trades_4strategy_alias(alias=alias,con=con) as_of_datetime = cu.convert_doubledate_2datetime(as_of_date) trades_frame = trades_frame[trades_frame['trade_date'] <= as_of_datetime] trades_frame['full_ticker'] = [trades_frame['ticker'].iloc[x] if trades_frame['instrument'].iloc[x] == 'F' else trades_frame['ticker'].iloc[x] + '_' + trades_frame['option_type'].iloc[x] + str(trades_frame['strike_price'].iloc[x]) for x in range(len(trades_frame.index))] grouped = trades_frame.groupby('full_ticker') net_position = pd.DataFrame() net_position['ticker'] = (grouped['ticker'].first()).values net_position['option_type'] = (grouped['option_type'].first()).values net_position['strike_price'] = (grouped['strike_price'].first()).values net_position['instrument'] = (grouped['instrument'].first()).values net_position['qty'] = (grouped['trade_quantity'].sum()).values net_position['qty'] = net_position['qty'].round(2) if 'con' not in kwargs.keys(): con.close() return net_position[net_position['qty'] != 0]
def generate_futures_butterfly_sheet_4date(**kwargs): date_to = kwargs['date_to'] output_dir = ts.create_strategy_output_dir(strategy_class='futures_butterfly', report_date=date_to) if os.path.isfile(output_dir + '/summary.pkl'): butterflies = pd.read_pickle(output_dir + '/summary.pkl') return {'butterflies': butterflies,'success': True} if 'volume_filter' not in kwargs.keys(): kwargs['volume_filter'] = 100 butterflies = get_futures_butterflies_4date(**kwargs) butterflies = butterflies[butterflies['trDte1'] >= 35] butterflies.reset_index(drop=True,inplace=True) num_butterflies = len(butterflies) q_list = [None]*num_butterflies qf_list = [None]*num_butterflies zscore1_list = [None]*num_butterflies zscore2_list = [None]*num_butterflies zscore3_list = [None]*num_butterflies zscore4_list = [None]*num_butterflies zscore5_list = [None]*num_butterflies zscore6_list = [None]*num_butterflies zscore7_list = [None]*num_butterflies rsquared1_list = [None]*num_butterflies rsquared2_list = [None]*num_butterflies regime_change_list = [None]*num_butterflies contract_seasonality_list = [None]*num_butterflies yield1_list = [None]*num_butterflies yield2_list = [None]*num_butterflies bf_price_list = [None]*num_butterflies bf_price_sell_limit_list = [None]*num_butterflies bf_price_buy_limit_list = [None]*num_butterflies noise_ratio_list = [None]*num_butterflies alpha1_list = [None]*num_butterflies alpha2_list = [None]*num_butterflies residual_std1_list = [None]*num_butterflies residual_std2_list = [None]*num_butterflies second_spread_weight_1_list = [None]*num_butterflies second_spread_weight_2_list = [None]*num_butterflies weight1_list = [None]*num_butterflies weight2_list = [None]*num_butterflies weight3_list = [None]*num_butterflies downside_list = [None]*num_butterflies upside_list = [None]*num_butterflies recent_5day_pnl_list = [None]*num_butterflies recent_vol_ratio_list = [None]*num_butterflies theo_pnl_list = [None]*num_butterflies theo_pnl5_list = [None]*num_butterflies theo_pnl10_list = [None]*num_butterflies theo_pnl15_list = [None]*num_butterflies theo_pnl20_list = [None]*num_butterflies theo_pnl25_list = [None]*num_butterflies ratio_target5_list = [None]*num_butterflies ratio_target10_list = [None]*num_butterflies ratio_target15_list = [None]*num_butterflies ratio_target20_list = [None]*num_butterflies ratio_target25_list = [None]*num_butterflies price_1_list = [None]*num_butterflies price_2_list = [None]*num_butterflies price_3_list = [None]*num_butterflies mean_reversion_rsquared_list = [None]*num_butterflies mean_reversion_signif_list = [None]*num_butterflies futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in cmi.futures_butterfly_strategy_tickerhead_list} date5_years_ago = cu.doubledate_shift(date_to,5*365) datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago) for i in range(num_butterflies): bf_signals_output = fs.get_futures_butterfly_signals(ticker_list=[butterflies['ticker1'][i], butterflies['ticker2'][i], butterflies['ticker3'][i]], tr_dte_list=[butterflies['trDte1'][i], butterflies['trDte2'][i], butterflies['trDte3'][i]], aggregation_method=butterflies['agg'][i], contracts_back=butterflies['cBack'][i], date_to=date_to, futures_data_dictionary=futures_data_dictionary, contract_multiplier=butterflies['multiplier'][i], datetime5_years_ago=datetime5_years_ago) q_list[i] = bf_signals_output['q'] qf_list[i] = bf_signals_output['qf'] zscore1_list[i] = bf_signals_output['zscore1'] zscore2_list[i] = bf_signals_output['zscore2'] zscore3_list[i] = bf_signals_output['zscore3'] zscore4_list[i] = bf_signals_output['zscore4'] zscore5_list[i] = bf_signals_output['zscore5'] zscore6_list[i] = bf_signals_output['zscore6'] zscore7_list[i] = bf_signals_output['zscore7'] rsquared1_list[i] = bf_signals_output['rsquared1'] rsquared2_list[i] = bf_signals_output['rsquared2'] regime_change_list[i] = bf_signals_output['regime_change_ind'] contract_seasonality_list[i] = bf_signals_output['contract_seasonality_ind'] yield1_list[i] = bf_signals_output['yield1_current'] yield2_list[i] = bf_signals_output['yield2_current'] bf_price_list[i] = bf_signals_output['bf_price'] bf_price_sell_limit_list[i] = bf_signals_output['short_price_limit'] bf_price_buy_limit_list[i] = bf_signals_output['long_price_limit'] noise_ratio_list[i] = bf_signals_output['noise_ratio'] alpha1_list[i] = bf_signals_output['alpha1'] alpha2_list[i] = bf_signals_output['alpha2'] residual_std1_list = bf_signals_output['residual_std1'] residual_std2_list = bf_signals_output['residual_std2'] second_spread_weight_1_list[i] = bf_signals_output['second_spread_weight_1'] second_spread_weight_2_list[i] = bf_signals_output['second_spread_weight_2'] weight1_list[i] = bf_signals_output['weight1'] weight2_list[i] = bf_signals_output['weight2'] weight3_list[i] = bf_signals_output['weight3'] downside_list[i] = bf_signals_output['downside'] upside_list[i] = bf_signals_output['upside'] recent_5day_pnl_list[i] = bf_signals_output['recent_5day_pnl'] recent_vol_ratio_list[i] = bf_signals_output['recent_vol_ratio'] theo_pnl_list[i] = bf_signals_output['theo_pnl'] theo_pnl5_list[i] = bf_signals_output['theo_pnl_list'][0] theo_pnl10_list[i] = bf_signals_output['theo_pnl_list'][1] theo_pnl15_list[i] = bf_signals_output['theo_pnl_list'][2] theo_pnl20_list[i] = bf_signals_output['theo_pnl_list'][3] theo_pnl25_list[i] = bf_signals_output['theo_pnl_list'][4] ratio_target5_list[i] = bf_signals_output['ratio_target_list'][0] ratio_target10_list[i] = bf_signals_output['ratio_target_list'][1] ratio_target15_list[i] = bf_signals_output['ratio_target_list'][2] ratio_target20_list[i] = bf_signals_output['ratio_target_list'][3] ratio_target25_list[i] = bf_signals_output['ratio_target_list'][4] price_1_list[i] = bf_signals_output['price_1'] price_2_list[i] = bf_signals_output['price_2'] price_3_list[i] = bf_signals_output['price_3'] mean_reversion_rsquared_list[i] = bf_signals_output['mean_reversion_rsquared'] mean_reversion_signif_list[i] = bf_signals_output['mean_reversion_signif'] butterflies['Q'] = q_list butterflies['QF'] = qf_list butterflies['z1'] = zscore1_list butterflies['z2'] = zscore2_list butterflies['z3'] = zscore3_list butterflies['z4'] = zscore4_list butterflies['z5'] = zscore5_list butterflies['z6'] = zscore6_list butterflies['z7'] = zscore7_list butterflies['r1'] = rsquared1_list butterflies['r2'] = rsquared2_list butterflies['RC'] = regime_change_list butterflies['seasonality'] = contract_seasonality_list butterflies['yield1'] = yield1_list butterflies['yield2'] = yield2_list butterflies['bf_price'] = bf_price_list butterflies['bf_sell_limit'] = bf_price_sell_limit_list butterflies['bf_buy_limit'] = bf_price_buy_limit_list butterflies['noise_ratio'] = noise_ratio_list butterflies['alpha1'] = alpha1_list butterflies['alpha2'] = alpha2_list butterflies['residual_std1'] = residual_std1_list butterflies['residual_std2'] = residual_std2_list butterflies['second_spread_weight_1'] = second_spread_weight_1_list butterflies['second_spread_weight_2'] = second_spread_weight_2_list butterflies['weight1'] = weight1_list butterflies['weight2'] = weight2_list butterflies['weight3'] = weight3_list butterflies['downside'] = downside_list butterflies['upside'] = upside_list butterflies['recent_5day_pnl'] = recent_5day_pnl_list butterflies['recent_vol_ratio'] = recent_vol_ratio_list butterflies['theo_pnl'] = theo_pnl_list butterflies['theo_pnl5'] = theo_pnl5_list butterflies['theo_pnl10'] = theo_pnl10_list butterflies['theo_pnl15'] = theo_pnl15_list butterflies['theo_pnl20'] = theo_pnl20_list butterflies['theo_pnl25'] = theo_pnl25_list butterflies['ratio_target5'] = ratio_target5_list butterflies['ratio_target10'] = ratio_target10_list butterflies['ratio_target15'] = ratio_target15_list butterflies['ratio_target20'] = ratio_target20_list butterflies['ratio_target25'] = ratio_target25_list butterflies['price1'] = price_1_list butterflies['price2'] = price_2_list butterflies['price3'] = price_3_list butterflies['mean_reversion_rsquared'] = mean_reversion_rsquared_list butterflies['mean_reversion_signif'] = mean_reversion_signif_list butterflies['z1'] = butterflies['z1'].round(2) butterflies['z2'] = butterflies['z2'].round(2) butterflies['z3'] = butterflies['z3'].round(2) butterflies['z4'] = butterflies['z4'].round(2) butterflies['z5'] = butterflies['z5'].round(2) butterflies['z6'] = butterflies['z6'].round(2) butterflies['z7'] = butterflies['z7'].round(2) butterflies['r1'] = butterflies['r1'].round(2) butterflies['r2'] = butterflies['r2'].round(2) butterflies['RC'] = butterflies['RC'].round(2) butterflies['seasonality'] = butterflies['seasonality'].round(2) butterflies['second_spread_weight_1'] = butterflies['second_spread_weight_1'].round(2) butterflies['second_spread_weight_2'] = butterflies['second_spread_weight_1'].round(2) butterflies['yield1'] = butterflies['yield1'].round(3) butterflies['yield2'] = butterflies['yield2'].round(3) butterflies['noise_ratio'] = butterflies['noise_ratio'].round(3) butterflies['alpha1'] = butterflies['alpha1'].round(3) butterflies['alpha2'] = butterflies['alpha2'].round(3) butterflies['residual_std1'] = butterflies['residual_std1'].round(3) butterflies['residual_std2'] = butterflies['residual_std2'].round(3) butterflies['downside'] = butterflies['downside'].round(3) butterflies['upside'] = butterflies['upside'].round(3) butterflies['recent_5day_pnl'] = butterflies['recent_5day_pnl'].round(3) butterflies['recent_vol_ratio'] = butterflies['recent_vol_ratio'].round(2) butterflies['theo_pnl'] = butterflies['theo_pnl'].round(3) butterflies['price1'] = butterflies['price1'].round(4) butterflies['price2'] = butterflies['price2'].round(4) butterflies['price3'] = butterflies['price3'].round(4) butterflies['mean_reversion_rsquared'] = butterflies['mean_reversion_rsquared'].round(2) butterflies.to_pickle(output_dir + '/summary.pkl') return {'butterflies': butterflies,'success': True}
def get_aligned_option_indicators_legacy(**kwargs): ticker_list = kwargs['ticker_list'] tr_dte_list = kwargs['tr_dte_list'] settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date']) if 'num_cal_days_back' in kwargs.keys(): num_cal_days_back = kwargs['num_cal_days_back'] else: num_cal_days_back = 20*365 settle_datetime_from = settle_datetime-dt.timedelta(num_cal_days_back) contract_specs_output_list = [cmi.get_contract_specs(x) for x in ticker_list] ticker_head_list = [x['ticker_head'] for x in contract_specs_output_list] cont_indx_list = [x['ticker_year']*100+x['ticker_month_num'] for x in contract_specs_output_list] month_seperation_list = [cmi.get_month_seperation_from_cont_indx(x,cont_indx_list[0]) for x in cont_indx_list] aggregation_method = max([ocu.get_aggregation_method_contracts_back({'ticker_class': x['ticker_class'], 'ticker_head': x['ticker_head']})['aggregation_method'] for x in contract_specs_output_list]) if (min(tr_dte_list) >= 80) and (aggregation_method == 1): aggregation_method = 3 tr_days_half_band_width_selected = ocu.tr_days_half_band_with[aggregation_method] data_frame_list = [] for x in range(len(ticker_list)): if ticker_head_list[x] in ['ED', 'E0', 'E2', 'E3', 'E4', 'E5']: model = 'OU' else: model = 'BS' tr_dte_upper_band = tr_dte_list[x]+tr_days_half_band_width_selected tr_dte_lower_band = tr_dte_list[x]-tr_days_half_band_width_selected ref_tr_dte_list = [y for y in cmi.aligned_data_tr_dte_list if y <= tr_dte_upper_band and y>=tr_dte_lower_band] if len(ref_tr_dte_list) == 0: return {'hist': [], 'current': [], 'success': False} if aggregation_method == 12: aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]], tr_dte_center=y, contract_month_letter=contract_specs_output_list[x]['ticker_month_str'], model=model) for y in ref_tr_dte_list] else: aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]], tr_dte_center=y, model=model) for y in ref_tr_dte_list] aligned_data = [y[(y['trDTE'] >= tr_dte_lower_band)&(y['trDTE'] <= tr_dte_upper_band)] for y in aligned_data] aligned_data = pd.concat(aligned_data) aligned_data.drop('theta', axis=1, inplace=True) aligned_data['settle_date'] = pd.to_datetime(aligned_data['settleDates'].astype('str'), format='%Y%m%d') aligned_data = aligned_data[(aligned_data['settle_date'] <= settle_datetime)&(aligned_data['settle_date'] >= settle_datetime_from)] aligned_data.rename(columns={'TickerYear': 'ticker_year', 'TickerMonth': 'ticker_month', 'trDTE': 'tr_dte', 'calDTE': 'cal_dte', 'impVol': 'imp_vol', 'close2CloseVol20': 'close2close_vol20', 'dollarTheta': 'theta'}, inplace=True) aligned_data.sort(['settle_date', 'ticker_year', 'ticker_month'], ascending=[True,True,True],inplace=True) aligned_data.drop_duplicates(['settle_date','ticker_year','ticker_month'],inplace=True) aligned_data = aligned_data[['settle_date','ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'theta', 'close2close_vol20', 'profit5']] aligned_data['cont_indx'] = 100*aligned_data['ticker_year']+aligned_data['ticker_month'] aligned_data['cont_indx_adj'] = [cmi.get_cont_indx_from_month_seperation(y,-month_seperation_list[x]) for y in aligned_data['cont_indx']] data_frame_list.append(aligned_data) for x in range(len(ticker_list)): data_frame_list[x].set_index(['settle_date','cont_indx_adj'], inplace=True,drop=False) merged_dataframe = pd.concat(data_frame_list, axis=1, join='inner',keys=['c'+ str(x+1) for x in range(len(ticker_list))]) merged_dataframe['abs_tr_dte_diff'] = abs(merged_dataframe['c1']['tr_dte']-tr_dte_list[0]) merged_dataframe['settle_date'] = merged_dataframe['c1']['settle_date'] merged_dataframe.sort(['settle_date', 'abs_tr_dte_diff'], ascending=[True,True], inplace=False) merged_dataframe.drop_duplicates('settle_date', inplace=True, take_last=False) merged_dataframe.index = merged_dataframe.index.droplevel(1) tr_dte_list = [] cal_dte_list = [] imp_vol_list = [] theta_list = [] close2close_vol20_list = [] for x in range(len(ticker_list)): selected_data = merged_dataframe['c' + str(x+1)] if settle_datetime in selected_data.index: selected_data = selected_data.loc[settle_datetime] else: return {'hist': [], 'current': [], 'success': False} if selected_data['cont_indx'] != cont_indx_list[x]: return {'hist': [], 'current': [], 'success': False} tr_dte_list.append(selected_data['tr_dte']) cal_dte_list.append(selected_data['cal_dte']) imp_vol_list.append(selected_data['imp_vol']) theta_list.append(selected_data['theta']) close2close_vol20_list.append(selected_data['close2close_vol20']) current_data = pd.DataFrame.from_items([('ticker',ticker_list), ('tr_dte', tr_dte_list), ('cal_dte', cal_dte_list), ('imp_vol', imp_vol_list), ('theta', theta_list), ('close2close_vol20', close2close_vol20_list)]) current_data['settle_date'] = settle_datetime current_data.set_index('ticker', drop=True, inplace=True) return {'hist': merged_dataframe, 'current': current_data, 'success': True}
def get_strategy_pnl_4day(**kwargs): alias = kwargs['alias'] pnl_date = kwargs['pnl_date'] #print(pnl_date) pnl_datetime = cu.convert_doubledate_2datetime(pnl_date) con = msu.get_my_sql_connection(**kwargs) if 'trades_frame' in kwargs.keys(): trades_frame = kwargs['trades_frame'] ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in trades_frame['ticker']] else: trades_frame = ts.get_trades_4strategy_alias(alias=alias,con=con) ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in trades_frame['ticker']] trades_frame['contract_multiplier'] = [cmi.contract_multiplier[x] for x in ticker_head_list] trades_frame['t_cost'] = [cmi.t_cost[x] for x in ticker_head_list] trades_frame['ticker_head'] = ticker_head_list if 'futures_data_dictionary' in kwargs.keys(): futures_data_dictionary = kwargs['futures_data_dictionary'] else: unique_ticker_head_list = list(set(ticker_head_list)) futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in unique_ticker_head_list} pnl_date_1 = exp.doubledate_shift_bus_days(double_date=pnl_date) underlying_frame = trades_frame[trades_frame['instrument'] == 'F'] option_frame = trades_frame[trades_frame['instrument'] == 'O'] futures_price_out_1 = [gfp.get_futures_price_preloaded(ticker=x, futures_data_dictionary=futures_data_dictionary, settle_date=pnl_date_1) for x in underlying_frame['ticker']] futures_price_out = [gfp.get_futures_price_preloaded(ticker=x, futures_data_dictionary=futures_data_dictionary, settle_date=pnl_date) for x in underlying_frame['ticker']] underlying_frame['price_1'] = [np.NaN if x.empty else x['close_price'].values[0] for x in futures_price_out_1] underlying_frame['price'] = [np.NaN if x.empty else x['close_price'].values[0] for x in futures_price_out] #underlying_frame['price_1'] = [gfp.get_futures_price_preloaded(ticker=x, # futures_data_dictionary=futures_data_dictionary, # settle_date=pnl_date_1)['close_price'].values[0] for x in underlying_frame['ticker']] #underlying_frame['price'] = [gfp.get_futures_price_preloaded(ticker=x, # futures_data_dictionary=futures_data_dictionary, # settle_date=pnl_date)['close_price'].values[0] for x in underlying_frame['ticker']] option_frame['price_1'] = [gop.get_options_price_from_db(ticker=option_frame['ticker'].iloc[x], strike=option_frame['strike_price'].iloc[x], option_type=option_frame['option_type'].iloc[x], con=con,settle_date=pnl_date_1, return_nan_if_emptyQ = True)['close_price'].values[0] for x in range(len(option_frame.index))] option_frame['price'] = [gop.get_options_price_from_db(ticker=option_frame['ticker'].iloc[x], strike=option_frame['strike_price'].iloc[x], option_type=option_frame['option_type'].iloc[x], con=con,settle_date=pnl_date, return_nan_if_emptyQ = True)['close_price'].values[0] for x in range(len(option_frame.index))] trades_frame = pd.concat([option_frame, underlying_frame]) position_frame = trades_frame[trades_frame['trade_date'] < pnl_datetime] intraday_frame = trades_frame[trades_frame['trade_date'] == pnl_datetime] position_pnl_per_ticker = pd.DataFrame(columns=['ticker','pnl_position']) intraday_pnl_per_ticker = pd.DataFrame(columns=['ticker','pnl_intraday']) position_pnl_per_tickerhead = pd.DataFrame(columns=['ticker_head','pnl_position']) intraday_pnl_per_tickerhead = pd.DataFrame(columns=['ticker_head','pnl_intraday']) if len(position_frame) == 0: position_pnl = 0 else: position_frame['pnl'] = position_frame['contract_multiplier']*\ position_frame['trade_quantity']*\ (position_frame['price']-position_frame['price_1']) position_pnl = position_frame['pnl'].sum() position_grouped_per_ticker = position_frame.groupby('ticker') position_grouped_per_tickerhead = position_frame.groupby('ticker_head') position_pnl_per_ticker['pnl_position'] = (position_grouped_per_ticker['pnl'].sum()).values position_pnl_per_ticker['ticker'] = (position_grouped_per_ticker['ticker'].first()).values position_pnl_per_tickerhead['pnl_position'] = (position_grouped_per_tickerhead['pnl'].sum()).values position_pnl_per_tickerhead['ticker_head'] = (position_grouped_per_tickerhead['ticker_head'].first()).values if len(intraday_frame) == 0: intraday_pnl = 0 t_cost = 0 else: intraday_frame['pnl'] = intraday_frame['contract_multiplier']*\ intraday_frame['trade_quantity']*\ (intraday_frame['price']-intraday_frame['trade_price']) intraday_frame['pnl_wtcost'] = intraday_frame['pnl']-abs(intraday_frame['trade_quantity']*intraday_frame['t_cost']) intraday_pnl = intraday_frame['pnl'].sum() t_cost = (abs(intraday_frame['trade_quantity']*intraday_frame['t_cost'])).sum() intraday_grouped_per_ticker = intraday_frame.groupby('ticker') intraday_grouped_per_tickerhead = intraday_frame.groupby('ticker_head') intraday_pnl_per_ticker['pnl_intraday'] = (intraday_grouped_per_ticker['pnl_wtcost'].sum()).values intraday_pnl_per_ticker['ticker'] = (intraday_grouped_per_ticker['ticker'].first()).values intraday_pnl_per_tickerhead['pnl_intraday'] = (intraday_grouped_per_tickerhead['pnl_wtcost'].sum()).values intraday_pnl_per_tickerhead['ticker_head'] = (intraday_grouped_per_tickerhead['ticker_head'].first()).values pnl_per_ticker = pd.merge(position_pnl_per_ticker,intraday_pnl_per_ticker,how='outer',on='ticker') intraday_zero_indx = [x not in intraday_pnl_per_ticker['ticker'].values for x in pnl_per_ticker['ticker']] position_zero_indx = [x not in position_pnl_per_ticker['ticker'].values for x in pnl_per_ticker['ticker']] pnl_per_ticker['pnl_position'][position_zero_indx] = 0 pnl_per_ticker['pnl_intraday'][intraday_zero_indx] = 0 pnl_per_ticker['pnl_total'] = pnl_per_ticker['pnl_position']+pnl_per_ticker['pnl_intraday'] pnl_per_ticker.set_index('ticker', drop=True, inplace=True) pnl_per_tickerhead = pd.merge(position_pnl_per_tickerhead,intraday_pnl_per_tickerhead,how='outer',on='ticker_head') intraday_zero_indx = [x not in intraday_pnl_per_tickerhead['ticker_head'].values for x in pnl_per_tickerhead['ticker_head']] position_zero_indx = [x not in position_pnl_per_tickerhead['ticker_head'].values for x in pnl_per_tickerhead['ticker_head']] pnl_per_tickerhead['pnl_position'][position_zero_indx] = 0 pnl_per_tickerhead['pnl_intraday'][intraday_zero_indx] = 0 pnl_per_tickerhead['pnl_total'] = pnl_per_tickerhead['pnl_position']+pnl_per_tickerhead['pnl_intraday'] pnl_per_tickerhead.set_index('ticker_head', drop=True, inplace=True) if 'con' not in kwargs.keys(): con.close() return {'total_pnl': int(position_pnl+intraday_pnl - t_cost), 'position_pnl': int(position_pnl), 'intraday_pnl': int(intraday_pnl), 't_cost': int(t_cost), 'pnl_per_ticker': pnl_per_ticker, 'pnl_per_tickerhead':pnl_per_tickerhead}
def update_options_price_database_from_cme_files_4ticker(**kwargs): ticker = kwargs['ticker'] contract_specs_output = cmi.get_contract_specs(ticker) ticker_head = contract_specs_output['ticker_head'] ticker_month_num = contract_specs_output['ticker_month_num'] ticker_year = contract_specs_output['ticker_year'] if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] kwargs['report_date'] = settle_date else: settle_date = int(time.strftime('%Y%m%d')) kwargs['settle_date'] = settle_date kwargs['report_date'] = settle_date if not exp.is_business_day(double_date=settle_date, reference_tickerhead=ticker_head): return if 'expiration_date' in kwargs.keys(): expiration_date = kwargs['expiration_date'] else: expiration_date = exp.get_options_expiration(ticker) expiration_date = expiration_date.date() settle_datetime = cu.convert_doubledate_2datetime(settle_date) if 'cal_dte' in kwargs.keys(): cal_dte = kwargs['cal_dte'] else: cal_dte = (expiration_date-settle_datetime.date()).days if 'tr_dte' in kwargs.keys(): tr_dte = kwargs['tr_dte'] else: bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head)) dts = pd.date_range(start=settle_datetime, end=expiration_date, freq=bday_us) tr_dte = len([x for x in dts if x.to_datetime().date() < expiration_date]) data_vendor_id = 2 now = dt.datetime.now() con = msu.get_my_sql_connection(**kwargs) process_output = pco.process_cme_options_4ticker(**kwargs) if process_output['success']: settle_frame = process_output['settle_frame'] else: if 'con' not in kwargs.keys(): con.close() return column_names = settle_frame.columns.tolist() option_type_indx = column_names.index('option_type') strike_indx = column_names.index('strike') settle_indx = column_names.index('settle') volume_indx = column_names.index('volume') interest_indx = column_names.index('interest') tuples = [tuple([data_vendor_id, ticker_head, ticker_month_num, ticker_year, ticker, x[option_type_indx],x[strike_indx],settle_datetime.date(), cal_dte, tr_dte, now, now, None if np.isnan(x[settle_indx]) else x[settle_indx], None if np.isnan(x[volume_indx]) else x[volume_indx], None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in settle_frame.values] column_str = "data_vendor_id, ticker_head, ticker_month, ticker_year, ticker, " \ " option_type, strike, price_date, cal_dte, tr_dte, " \ " created_date,last_updated_date, close_price, volume, open_interest" insert_str = ("%s, " * len(column_str.split(',')))[:-2] final_str = "REPLACE INTO daily_option_price (%s) VALUES (%s)" % (column_str, insert_str) msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con) if 'con' not in kwargs.keys(): con.close()
def get_option_greeks(**kwargs): # risk_free_rate and volatility in whole points not percentage points # for example enter 0.02 for 2 percent interest rate # enter 0.25 for 25 percent annual volatility underlying = kwargs['underlying'] strike = kwargs['strike'] risk_free_rate = kwargs['risk_free_rate'] expiration_date = kwargs['expiration_date'] calculation_date = kwargs['calculation_date'] option_type = kwargs['option_type'].upper() exercise_type = kwargs['exercise_type'].upper() dividend_rate = 0 expiration_datetime = cu.convert_doubledate_2datetime(expiration_date) calculation_datetime = cu.convert_doubledate_2datetime(calculation_date) expiration_date_obj = ql.Date(expiration_datetime.day, expiration_datetime.month, expiration_datetime.year) calculation_date_obj = ql.Date(calculation_datetime.day, calculation_datetime.month, calculation_datetime.year) cal_dte = day_count_obj.dayCount(calculation_date_obj, expiration_date_obj) nan_greeks = {'option_price': np.NaN, 'implied_vol': np.NaN, 'delta': np.NaN, 'vega': np.NaN, 'theta': np.NaN, 'cal_dte': cal_dte, 'gamma': np.NaN} #print(underlying) #print(kwargs['option_price']) #print(option_type) #print(exercise_type) #print(risk_free_rate) #print(expiration_date) #print(calculation_date) #print(strike) if 'option_price' in kwargs.keys(): if option_type == 'C': if kwargs['option_price']+strike-underlying <= 10**(-12): nan_greeks['delta'] = 1 return nan_greeks elif option_type == 'P': if kwargs['option_price']-strike+underlying <= 10**(-12): nan_greeks['delta'] = -1 return nan_greeks if cal_dte == 0: if option_type == 'C': if strike <= underlying: nan_greeks['delta'] = 1 else: nan_greeks['delta'] = 0 elif option_type == 'P': if strike >= underlying: nan_greeks['delta'] = -1 else: nan_greeks['delta'] = 0 return nan_greeks if 'implied_vol' in kwargs.keys(): implied_vol = kwargs['implied_vol'] else: implied_vol = 0.15 if 'engine_name' in kwargs.keys(): engine_name = kwargs['engine_name'] else: engine_name = 'baw' if option_type == 'C': option_type_obj = ql.Option.Call elif option_type == 'P': option_type_obj = ql.Option.Put ql.Settings.instance().evaluationDate = calculation_date_obj if exercise_type == 'E': exercise_obj = ql.EuropeanExercise(expiration_date_obj) elif exercise_type == 'A': exercise_obj = ql.AmericanExercise(calculation_date_obj, expiration_date_obj) #print('years to expitation: ' + str(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj))) #print('spot: ' + str(underlying/m.exp(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj)*risk_free_rate))) #underlying_obj = ql.QuoteHandle(ql.SimpleQuote(underlying/m.exp(day_count_obj.yearFraction(calculation_date_obj, expiration_date_obj)*risk_free_rate))) underlying_obj = ql.QuoteHandle(ql.SimpleQuote(underlying)) flat_ts_obj = ql.YieldTermStructureHandle(ql.FlatForward(calculation_date_obj, risk_free_rate, day_count_obj)) dividend_yield_obj = ql.YieldTermStructureHandle(ql.FlatForward(calculation_date_obj, dividend_rate, day_count_obj)) flat_vol_ts_obj = ql.BlackVolTermStructureHandle(ql.BlackConstantVol(calculation_date_obj, calendar_obj, implied_vol, day_count_obj)) #bsm_process = ql.BlackScholesMertonProcess(underlying_obj, dividend_yield_obj, flat_ts_obj, flat_vol_ts_obj) bsm_process = ql.BlackProcess(underlying_obj, flat_ts_obj, flat_vol_ts_obj) payoff = ql.PlainVanillaPayoff(option_type_obj, strike) option_obj = ql.VanillaOption(payoff, exercise_obj) if engine_name == 'baw': option_obj.setPricingEngine(ql.BaroneAdesiWhaleyEngine(bsm_process)) elif engine_name == 'fda': option_obj.setPricingEngine(ql.FDAmericanEngine(bsm_process, 100, 100)) option_price = option_obj.NPV() if 'option_price' in kwargs.keys(): try: implied_vol = option_obj.impliedVolatility(targetValue=kwargs['option_price'], process=bsm_process,accuracy=0.00001) flat_vol_ts_obj = ql.BlackVolTermStructureHandle(ql.BlackConstantVol(calculation_date_obj, calendar_obj, implied_vol, day_count_obj)) #bsm_process = ql.BlackScholesMertonProcess(underlying_obj, dividend_yield_obj, flat_ts_obj, flat_vol_ts_obj) bsm_process = ql.BlackProcess(underlying_obj, flat_ts_obj, flat_vol_ts_obj) if engine_name == 'baw': option_obj.setPricingEngine(ql.BaroneAdesiWhaleyEngine(bsm_process)) elif engine_name == 'fda': option_obj.setPricingEngine(ql.FDAmericanEngine(bsm_process, 100, 100)) option_price = option_obj.NPV() except Exception: return nan_greeks option_obj = ql.VanillaOption(payoff, ql.EuropeanExercise(expiration_date_obj)) option_obj.setPricingEngine(ql.AnalyticEuropeanEngine(bsm_process)) return {'option_price': option_price, 'implied_vol': implied_vol, 'delta': option_obj.delta(), 'vega': option_obj.vega(), 'theta': option_obj.thetaPerDay(), 'cal_dte': cal_dte, 'gamma': option_obj.gamma()}
def get_curve_pca_report(**kwargs): ticker_head = kwargs['ticker_head'] date_to = kwargs['date_to'] if 'use_existing_filesQ' in kwargs.keys(): use_existing_filesQ = kwargs['use_existing_filesQ'] else: use_existing_filesQ = True output_dir = ts.create_strategy_output_dir(strategy_class='curve_pca', report_date=date_to) if os.path.isfile(output_dir + '/' + ticker_head + '.pkl') and use_existing_filesQ: pca_results = pd.read_pickle(output_dir + '/' + ticker_head + '.pkl') return {'pca_results': pca_results, 'success': True} date10_years_ago = cu.doubledate_shift(date_to, 10*365) datetime_to = cu.convert_doubledate_2datetime(date_to) if ticker_head == 'ED': num_contracts = 12 rolling_data = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_contracts, front_tr_dte_limit=10, date_from=date10_years_ago, date_to=date_to) if datetime_to != rolling_data[0].index[-1].to_datetime(): return {'pca_results': pd.DataFrame(), 'success': False} merged_data = pd.concat(rolling_data, axis=1, join='inner') total_range = list(range(len(rolling_data))) index_exclude = [len(total_range)-1] month_spread = [3]*(len(rolling_data)-1) elif ticker_head in ['CL', 'B']: num_monthly_contracts = 18 if ticker_head == 'CL': num_semiannual_contracts = 6 elif ticker_head == 'B': num_semiannual_contracts = 7 rolling_data_monthly = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_monthly_contracts, front_tr_dte_limit=10, date_from=date10_years_ago, date_to=date_to) rolling_data_semiannual = cd.get_rolling_curve_data(ticker_head=ticker_head, num_contracts=num_semiannual_contracts, front_tr_dte_limit=10, date_from=date10_years_ago, month_separation=6, date_to=date_to) if datetime_to != rolling_data_monthly[0].index[-1].to_datetime() or datetime_to != rolling_data_semiannual[0].index[-1].to_datetime(): return {'pca_results': pd.DataFrame(), 'success': False} rolling_data_merged = pd.concat(rolling_data_semiannual, axis=1) annual_select = rolling_data_merged['ticker_month'].iloc[-1] % 12 == 0 rolling_data_annual = [rolling_data_semiannual[x] for x in range(len(rolling_data_semiannual)) if annual_select.values[x]] merged_data = pd.concat(rolling_data_monthly+rolling_data_semiannual+rolling_data_annual, axis=1, join='inner') total_range = list(range(len(rolling_data_monthly)+len(rolling_data_semiannual)+len(rolling_data_annual))) index_exclude = [len(rolling_data_monthly)-1,len(rolling_data_monthly)+len(rolling_data_semiannual)-1, len(total_range)-1] month_spread = [1]*(len(rolling_data_monthly)-1)+[6]*(len(rolling_data_semiannual)-1)+[12]*(len(rolling_data_annual)-1) yield_raw = [(merged_data['close_price'].ix[:, x]-merged_data['close_price'].ix[:, x+1]) / merged_data['close_price'].ix[:, x+1] for x in total_range if x not in index_exclude] yield_merged = pd.concat(yield_raw, axis=1) yield_data = 100*yield_merged.values change5_raw = [(merged_data['change5'].ix[:, x]-merged_data['change5'].ix[:, x+1]) for x in total_range if x not in index_exclude] change5_merged = pd.concat(change5_raw, axis=1) change5_data = change5_merged.values change10_raw = [(merged_data['change10'].ix[:, x]-merged_data['change10'].ix[:, x+1]) for x in total_range if x not in index_exclude] change10_merged = pd.concat(change10_raw, axis=1) change10_data = change10_merged.values change20_raw = [(merged_data['change20'].ix[:, x]-merged_data['change20'].ix[:, x+1]) for x in total_range if x not in index_exclude] change20_merged = pd.concat(change20_raw, axis=1) change20_data = change20_merged.values tr_dte_raw = [merged_data['tr_dte'].ix[:, x] for x in total_range if x not in index_exclude] tr_dte_merged = pd.concat(tr_dte_raw, axis=1) tr_dte_data = tr_dte_merged.values ticker_month_raw = [merged_data['ticker_month'].ix[:, x] for x in total_range if x not in index_exclude] ticker_month_merged = pd.concat(ticker_month_raw, axis=1) ticker_month_data = ticker_month_merged.values ticker1_list = [merged_data['ticker'].ix[-1, x] for x in total_range if x not in index_exclude] ticker2_list = [merged_data['ticker'].ix[-1, x+1] for x in total_range if x not in index_exclude] price_list = [(merged_data['close_price'].ix[-1, x]-merged_data['close_price'].ix[-1, x+1]) for x in total_range if x not in index_exclude] pca_out = stats.get_pca(data_input=yield_data, n_components=2) residuals = yield_data-pca_out['model_fit'] pca_results = pd.DataFrame.from_items([('ticker1',ticker1_list), ('ticker2',ticker2_list), ('monthSpread',month_spread), ('tr_dte_front', tr_dte_data[-1]), ('ticker_month_front', ticker_month_data[-1]), ('residuals',residuals[-1]), ('price',price_list), ('yield',yield_data[-1]), ('z', (residuals[-1]-residuals.mean(axis=0))/residuals.std(axis=0)), ('factor_load1',pca_out['loadings'][0]), ('factor_load2',pca_out['loadings'][1]), ('change5', change5_data[-1]), ('change10', change10_data[-1]), ('change20', change20_data[-1])]) # notice that this date_to needs to me removed once we are done with backtesting seasonality_adjustment = fs.get_pca_seasonality_adjustments(ticker_head=ticker_head,date_to=date_to) pca_results = pd.merge(pca_results, seasonality_adjustment,how='left',on=['monthSpread','ticker_month_front']) pca_results['z2'] = pca_results['z']-pca_results['z_seasonal_mean'] pca_results['residuals'] = pca_results['residuals'].round(3) pca_results['yield'] = pca_results['yield'].round(2) pca_results['z'] = pca_results['z'].round(2) pca_results['z2'] = pca_results['z2'].round(2) pca_results['z_seasonal_mean'] = pca_results['z_seasonal_mean'].round(2) pca_results['factor_load1'] = pca_results['factor_load1'].round(3) pca_results['factor_load2'] = pca_results['factor_load2'].round(3) pca_results.to_pickle(output_dir + '/' + ticker_head + '.pkl') return {'pca_results': pca_results, 'success': True}
def get_aligned_option_indicators(**kwargs): ticker_list = kwargs['ticker_list'] settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date']) #print(ticker_list) if 'num_cal_days_back' in kwargs.keys(): num_cal_days_back = kwargs['num_cal_days_back'] else: num_cal_days_back = 20*365 settle_datetime_from = settle_datetime-dt.timedelta(num_cal_days_back) contract_specs_output_list = [cmi.get_contract_specs(x) for x in ticker_list] ticker_head_list = [x['ticker_head'] for x in contract_specs_output_list] contract_multiplier_list = [cmi.contract_multiplier[x['ticker_head']] for x in contract_specs_output_list] cont_indx_list = [x['ticker_year']*100+x['ticker_month_num'] for x in contract_specs_output_list] month_seperation_list = [cmi.get_month_seperation_from_cont_indx(x,cont_indx_list[0]) for x in cont_indx_list] if 'option_ticker_indicator_dictionary' in kwargs.keys(): option_ticker_indicator_dictionary = kwargs['option_ticker_indicator_dictionary'] else: con = msu.get_my_sql_connection(**kwargs) unique_ticker_heads = list(set(ticker_head_list)) option_ticker_indicator_dictionary = {x: get_option_ticker_indicators(ticker_head=x, settle_date_to=kwargs['settle_date'], num_cal_days_back=num_cal_days_back, con=con) for x in unique_ticker_heads} if 'con' not in kwargs.keys(): con.close() option_ticker_indicator_dictionary_final = {ticker_list[x]: option_ticker_indicator_dictionary[ticker_head_list[x]] for x in range(len(ticker_list))} max_available_settle_list = [] tr_dte_list = [] cal_dte_list = [] imp_vol_list = [] theta_list = [] close2close_vol20_list = [] volume_list = [] open_interest_list = [] for x in range(len(ticker_list)): ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]] ticker_data = ticker_data[ticker_data['settle_date'] <= settle_datetime] option_ticker_indicator_dictionary_final[ticker_list[x]] = ticker_data ticker_data = ticker_data[ticker_data['ticker'] == ticker_list[x]] max_available_settle_list.append(ticker_data['settle_date'].iloc[-1]) last_available_settle = min(max_available_settle_list) for x in range(len(ticker_list)): ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]] ticker_data = ticker_data[(ticker_data['ticker'] == ticker_list[x]) & (ticker_data['settle_date'] == last_available_settle)] tr_dte_list.append(ticker_data['tr_dte'].iloc[0]) cal_dte_list.append(ticker_data['cal_dte'].iloc[0]) imp_vol_list.append(ticker_data['imp_vol'].iloc[0]) theta_list.append(ticker_data['theta'].iloc[0]*contract_multiplier_list[x]) close2close_vol20_list.append(ticker_data['close2close_vol20'].iloc[0]) volume_list.append(ticker_data['volume'].iloc[0]) open_interest_list.append(ticker_data['open_interest'].iloc[0]) current_data = pd.DataFrame.from_items([('ticker',ticker_list), ('tr_dte', tr_dte_list), ('cal_dte', cal_dte_list), ('imp_vol', imp_vol_list), ('theta', theta_list), ('close2close_vol20', close2close_vol20_list), ('volume', volume_list), ('open_interest', open_interest_list)]) current_data['settle_date'] = last_available_settle current_data.set_index('ticker', drop=True, inplace=True) current_data = current_data[['settle_date', 'tr_dte', 'cal_dte', 'imp_vol', 'close2close_vol20', 'theta', 'volume', 'open_interest']] aggregation_method = max([ocu.get_aggregation_method_contracts_back({'ticker_class': x['ticker_class'], 'ticker_head': x['ticker_head']})['aggregation_method'] for x in contract_specs_output_list]) if (current_data['tr_dte'].min() >= 80) and (aggregation_method == 1): aggregation_method = 3 tr_days_half_band_width_selected = ocu.tr_days_half_band_with[aggregation_method] data_frame_list = [] ref_tr_dte_list_list = [] for x in range(len(ticker_list)): ticker_data = option_ticker_indicator_dictionary_final[ticker_list[x]] if ticker_head_list[x] in ['ED', 'E0', 'E2', 'E3', 'E4', 'E5']: model = 'OU' else: model = 'BS' tr_dte_upper_band = current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected tr_dte_lower_band = current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected ref_tr_dte_list = [y for y in cmi.aligned_data_tr_dte_list if y <= tr_dte_upper_band and y>=tr_dte_lower_band] if len(ref_tr_dte_list) == 0: return {'hist': [], 'current': [], 'success': False} if aggregation_method == 12: aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]], tr_dte_center=y, contract_month_letter=contract_specs_output_list[x]['ticker_month_str'], model=model) for y in ref_tr_dte_list] ticker_data = ticker_data[ticker_data['ticker_month'] == contract_specs_output_list[x]['ticker_month_num']] else: aligned_data = [gop.load_aligend_options_data_file(ticker_head=cmi.aligned_data_tickerhead[ticker_head_list[x]], tr_dte_center=y, model=model) for y in ref_tr_dte_list] aligned_data = [y[(y['trDTE'] >= current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected)& (y['trDTE'] <= current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected)] for y in aligned_data] aligned_data = pd.concat(aligned_data) aligned_data['settle_date'] = pd.to_datetime(aligned_data['settleDates'].astype('str'), format='%Y%m%d') aligned_data.rename(columns={'TickerYear': 'ticker_year', 'TickerMonth': 'ticker_month', 'trDTE': 'tr_dte', 'calDTE': 'cal_dte', 'impVol': 'imp_vol', 'close2CloseVol20': 'close2close_vol20'}, inplace=True) aligned_data.sort(['settle_date', 'ticker_year', 'ticker_month'], ascending=[True,True,True],inplace=True) aligned_data.drop_duplicates(['settle_date','ticker_year','ticker_month'],inplace=True) aligned_data['old_aligned'] = True aligned_data = aligned_data[['settle_date','ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'close2close_vol20', 'profit5', 'old_aligned']] tr_dte_selection = (ticker_data['tr_dte'] >= current_data['tr_dte'].loc[ticker_list[x]]-tr_days_half_band_width_selected)&\ (ticker_data['tr_dte'] <= current_data['tr_dte'].loc[ticker_list[x]]+tr_days_half_band_width_selected) ticker_data = ticker_data[tr_dte_selection] ticker_data['old_aligned'] = False ticker_data['profit5'] = np.NaN ticker_data = pd.concat([aligned_data, ticker_data[['settle_date', 'ticker_month', 'ticker_year', 'cal_dte', 'tr_dte', 'imp_vol', 'close2close_vol20', 'profit5', 'old_aligned']]]) ticker_data = ticker_data[(ticker_data['settle_date'] <= settle_datetime)&(ticker_data['settle_date'] >= settle_datetime_from)] ticker_data['cont_indx'] = 100*ticker_data['ticker_year']+ticker_data['ticker_month'] ticker_data['cont_indx_adj'] = [cmi.get_cont_indx_from_month_seperation(y,-month_seperation_list[x]) for y in ticker_data['cont_indx']] data_frame_list.append(ticker_data) ref_tr_dte_list_list.append(ref_tr_dte_list) for x in range(len(ticker_list)): data_frame_list[x].set_index(['settle_date','cont_indx_adj'], inplace=True,drop=False) data_frame_list[x]['imp_vol'] = data_frame_list[x]['imp_vol'].astype('float64') merged_dataframe = pd.concat(data_frame_list, axis=1, join='inner',keys=['c'+ str(x+1) for x in range(len(ticker_list))]) merged_dataframe['abs_tr_dte_diff'] = abs(merged_dataframe['c1']['tr_dte']-tr_dte_list[0]) merged_dataframe['settle_date'] = merged_dataframe['c1']['settle_date'] merged_dataframe.sort(['settle_date', 'abs_tr_dte_diff'], ascending=[True,True], inplace=False) merged_dataframe.drop_duplicates('settle_date', inplace=True, take_last=False) merged_dataframe.index = merged_dataframe.index.droplevel(1) return {'hist': merged_dataframe, 'current': current_data, 'success': True}
def get_vcs_pairs_4date_legacy(**kwargs): settle_date = kwargs['settle_date'] settle_datetime = cu.convert_doubledate_2datetime(settle_date) con = msu.get_my_sql_connection(**kwargs) liquid_options_frame = cl.generate_liquid_options_list_dataframe(settle_date=settle_date,con=con) contract_specs_output = [cmi.get_contract_specs(x) for x in liquid_options_frame['ticker']] liquid_options_frame['ticker_head'] = [x['ticker_head'] for x in contract_specs_output] liquid_options_frame['ticker_month'] = [x['ticker_month_num'] for x in contract_specs_output] liquid_options_frame['ticker_class'] = [x['ticker_class'] for x in contract_specs_output] liquid_options_frame['cal_dte'] = [(x-settle_datetime.date()).days for x in liquid_options_frame['expiration_date']] liquid_options_frame = liquid_options_frame[((liquid_options_frame['ticker_head'] == 'LN')&(liquid_options_frame['cal_dte'] <= 360))| ((liquid_options_frame['ticker_head']=='LC')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']%2==0))| ((liquid_options_frame['ticker_head']=='LC')&(liquid_options_frame['cal_dte']<=40)& (liquid_options_frame['ticker_month']%2==1))| ((liquid_options_frame['ticker_head']=='ES')&(liquid_options_frame['cal_dte']<=270))| ((liquid_options_frame['ticker_class']=='FX')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month']%3==0))| ((liquid_options_frame['ticker_class']=='FX')&(liquid_options_frame['cal_dte']<=70)& (liquid_options_frame['ticker_month']%3!=0))| ((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month'].isin([6,12])))| ((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month'].isin([2,4,8,10])))| ((liquid_options_frame['ticker_head']=='GC')&(liquid_options_frame['cal_dte']<=70))| ((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month'].isin([7,12])))| ((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month'].isin([1,3,5,9])))| ((liquid_options_frame['ticker_head']=='SI')&(liquid_options_frame['cal_dte']<=70))| ((liquid_options_frame['ticker_class']=='Treasury')&(liquid_options_frame['cal_dte']<=180)& (liquid_options_frame['ticker_month']%3==0))| ((liquid_options_frame['ticker_class']=='Treasury')&(liquid_options_frame['cal_dte']<=70)& (liquid_options_frame['ticker_month']%3!=0))| ((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=540)& (liquid_options_frame['ticker_month']==12))| ((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']==7))| ((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month'].isin([3,5,9])))| ((liquid_options_frame['ticker_head']=='C')&(liquid_options_frame['cal_dte']<=40))| ((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=540)& (liquid_options_frame['ticker_month']==11))| ((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']==7))| ((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month'].isin([1,3,5,8,9])))| ((liquid_options_frame['ticker_head']=='S')&(liquid_options_frame['cal_dte']<=40))| ((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']==12))| ((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month']==7))| ((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=180)& (liquid_options_frame['ticker_month'].isin([1,3,5,8,9,10])))| ((liquid_options_frame['ticker_head']=='SM')&(liquid_options_frame['cal_dte']<=40))| ((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']==12))| ((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month']==7))| ((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=180)& (liquid_options_frame['ticker_month'].isin([1,3,5,8,9,10])))| ((liquid_options_frame['ticker_head']=='BO')&(liquid_options_frame['cal_dte']<=40))| ((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=360)& (liquid_options_frame['ticker_month']==12))| ((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=270)& (liquid_options_frame['ticker_month']==7))| ((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=180)& (liquid_options_frame['ticker_month'].isin([3,5,9])))| ((liquid_options_frame['ticker_head']=='W')&(liquid_options_frame['cal_dte']<=40))| ((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=720)& (liquid_options_frame['ticker_month']==12))| ((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=540)& (liquid_options_frame['ticker_month']==6))| ((liquid_options_frame['ticker_head']=='CL')&(liquid_options_frame['cal_dte']<=180))| ((liquid_options_frame['ticker_head']=='NG')&(liquid_options_frame['cal_dte']<=360))] liquid_options_frame.sort(['ticker_head','cal_dte'],ascending=[True,True],inplace=True) liquid_options_frame['tr_dte'] = [exp.get_days2_expiration(date_to=settle_date,con=con,instrument='options',ticker=x)['tr_dte'] for x in liquid_options_frame['ticker']] if 'con' not in kwargs.keys(): con.close() option_frame = liquid_options_frame[liquid_options_frame['tr_dte'] >= 35] option_frame.reset_index(drop=True,inplace=True) unique_ticker_heads = option_frame['ticker_head'].unique() tuples = [] for ticker_head_i in unique_ticker_heads: ticker_head_data = option_frame[option_frame['ticker_head'] == ticker_head_i] ticker_head_data.sort('cal_dte', ascending=True, inplace=True) if len(ticker_head_data.index) >= 2: for i in range(len(ticker_head_data.index)-1): for j in range(i+1,len(ticker_head_data.index)): tuples = tuples + [(ticker_head_data.index[i], ticker_head_data.index[j])] return pd.DataFrame([(option_frame['ticker'][indx[0]], option_frame['ticker'][indx[1]], option_frame['ticker_head'][indx[0]], option_frame['ticker_class'][indx[0]], option_frame['tr_dte'][indx[0]], option_frame['tr_dte'][indx[1]]) for indx in tuples],columns=['ticker1','ticker2','tickerHead','tickerClass','trDte1','trDte2'])
def get_intraday_spread_signals(**kwargs): ticker_list = kwargs['ticker_list'] date_to = kwargs['date_to'] ticker_list = [x for x in ticker_list if x is not None] ticker_head_list = [cmi.get_contract_specs(x)['ticker_head'] for x in ticker_list] ticker_class_list = [cmi.ticker_class[x] for x in ticker_head_list] print('-'.join(ticker_list)) if 'tr_dte_list' in kwargs.keys(): tr_dte_list = kwargs['tr_dte_list'] else: tr_dte_list = [exp.get_days2_expiration(ticker=x,date_to=date_to, instrument='futures')['tr_dte'] for x in ticker_list] weights_output = sutil.get_spread_weights_4contract_list(ticker_head_list=ticker_head_list) if 'aggregation_method' in kwargs.keys() and 'contracts_back' in kwargs.keys(): aggregation_method = kwargs['aggregation_method'] contracts_back = kwargs['contracts_back'] else: amcb_output = [opUtil.get_aggregation_method_contracts_back(cmi.get_contract_specs(x)) for x in ticker_list] aggregation_method = max([x['aggregation_method'] for x in amcb_output]) contracts_back = min([x['contracts_back'] for x in amcb_output]) if 'futures_data_dictionary' in kwargs.keys(): futures_data_dictionary = kwargs['futures_data_dictionary'] else: futures_data_dictionary = {x: gfp.get_futures_price_preloaded(ticker_head=x) for x in list(set(ticker_head_list))} if 'use_last_as_current' in kwargs.keys(): use_last_as_current = kwargs['use_last_as_current'] else: use_last_as_current = True if 'datetime5_years_ago' in kwargs.keys(): datetime5_years_ago = kwargs['datetime5_years_ago'] else: date5_years_ago = cu.doubledate_shift(date_to,5*365) datetime5_years_ago = cu.convert_doubledate_2datetime(date5_years_ago) if 'num_days_back_4intraday' in kwargs.keys(): num_days_back_4intraday = kwargs['num_days_back_4intraday'] else: num_days_back_4intraday = 5 contract_multiplier_list = [cmi.contract_multiplier[x] for x in ticker_head_list] aligned_output = opUtil.get_aligned_futures_data(contract_list=ticker_list, tr_dte_list=tr_dte_list, aggregation_method=aggregation_method, contracts_back=contracts_back, date_to=date_to, futures_data_dictionary=futures_data_dictionary, use_last_as_current=use_last_as_current) aligned_data = aligned_output['aligned_data'] current_data = aligned_output['current_data'] spread_weights = weights_output['spread_weights'] portfolio_weights = weights_output['portfolio_weights'] aligned_data['spread'] = 0 aligned_data['spread_pnl_1'] = 0 aligned_data['spread_pnl1'] = 0 spread_settle = 0 last5_years_indx = aligned_data['settle_date']>=datetime5_years_ago num_contracts = len(ticker_list) for i in range(num_contracts): aligned_data['spread'] = aligned_data['spread']+aligned_data['c' + str(i+1)]['close_price']*spread_weights[i] spread_settle = spread_settle + current_data['c' + str(i+1)]['close_price']*spread_weights[i] aligned_data['spread_pnl_1'] = aligned_data['spread_pnl_1']+aligned_data['c' + str(i+1)]['change_1']*portfolio_weights[i]*contract_multiplier_list[i] aligned_data['spread_pnl1'] = aligned_data['spread_pnl1']+aligned_data['c' + str(i+1)]['change1_instant']*portfolio_weights[i]*contract_multiplier_list[i] aligned_data['spread_normalized'] = aligned_data['spread']/aligned_data['c1']['close_price'] data_last5_years = aligned_data[last5_years_indx] percentile_vector = stats.get_number_from_quantile(y=data_last5_years['spread_pnl_1'].values, quantile_list=[1, 15, 85, 99], clean_num_obs=max(100, round(3*len(data_last5_years.index)/4))) downside = (percentile_vector[0]+percentile_vector[1])/2 upside = (percentile_vector[2]+percentile_vector[3])/2 date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=x) for x in reversed(range(1,num_days_back_4intraday))] date_list.append(date_to) intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=ticker_list, date_list=date_list) intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index] intraday_data['settle_date'] = intraday_data['time_stamp'].apply(lambda x: x.date()) end_hour = min([cmi.last_trade_hour_minute[x] for x in ticker_head_list]) start_hour = max([cmi.first_trade_hour_minute[x] for x in ticker_head_list]) trade_start_hour = dt.time(9, 30, 0, 0) if 'Ag' in ticker_class_list: start_hour1 = dt.time(0, 45, 0, 0) end_hour1 = dt.time(7, 45, 0, 0) selection_indx = [x for x in range(len(intraday_data.index)) if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or ((intraday_data['time_stamp'].iloc[x].time() < end_hour) and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))] else: selection_indx = [x for x in range(len(intraday_data.index)) if (intraday_data.index[x].to_datetime().time() < end_hour) and(intraday_data.index[x].to_datetime().time() >= start_hour)] intraday_data = intraday_data.iloc[selection_indx] intraday_data['spread'] = 0 for i in range(num_contracts): intraday_data['c' + str(i+1), 'mid_p'] = (intraday_data['c' + str(i+1)]['best_bid_p'] + intraday_data['c' + str(i+1)]['best_ask_p'])/2 intraday_data['spread'] = intraday_data['spread']+intraday_data['c' + str(i+1)]['mid_p']*spread_weights[i] unique_settle_dates = intraday_data['settle_date'].unique() intraday_data['spread1'] = np.nan for i in range(len(unique_settle_dates)-1): if (intraday_data['settle_date'] == unique_settle_dates[i]).sum() == \ (intraday_data['settle_date'] == unique_settle_dates[i+1]).sum(): intraday_data.loc[intraday_data['settle_date'] == unique_settle_dates[i],'spread1'] = \ intraday_data['spread'][intraday_data['settle_date'] == unique_settle_dates[i+1]].values intraday_data = intraday_data[intraday_data['settle_date'].notnull()] intraday_mean = intraday_data['spread'].mean() intraday_std = intraday_data['spread'].std() intraday_data_last2days = intraday_data[intraday_data['settle_date'] >= cu.convert_doubledate_2datetime(date_list[-2]).date()] intraday_data_yesterday = intraday_data[intraday_data['settle_date'] == cu.convert_doubledate_2datetime(date_list[-1]).date()] intraday_mean2 = intraday_data_last2days['spread'].mean() intraday_std2 = intraday_data_last2days['spread'].std() intraday_mean1 = intraday_data_yesterday['spread'].mean() intraday_std1 = intraday_data_yesterday['spread'].std() intraday_z = (spread_settle-intraday_mean)/intraday_std num_obs_intraday = len(intraday_data.index) num_obs_intraday_half = round(num_obs_intraday/2) intraday_tail = intraday_data.tail(num_obs_intraday_half) num_positives = sum(intraday_tail['spread'] > intraday_data['spread'].mean()) num_negatives = sum(intraday_tail['spread'] < intraday_data['spread'].mean()) recent_trend = 100*(num_positives-num_negatives)/(num_positives+num_negatives) pnl_frame = ifs.get_pnl_4_date_range(date_to=date_to, num_bus_days_back=20, ticker_list=ticker_list) if (len(pnl_frame.index)>15)&(pnl_frame['total_pnl'].std() != 0): historical_sharp = (250**(0.5))*pnl_frame['total_pnl'].mean()/pnl_frame['total_pnl'].std() else: historical_sharp = np.nan return {'downside': downside, 'upside': upside,'intraday_data': intraday_data, 'z': intraday_z,'recent_trend': recent_trend, 'intraday_mean': intraday_mean, 'intraday_std': intraday_std, 'intraday_mean2': intraday_mean2, 'intraday_std2': intraday_std2, 'intraday_mean1': intraday_mean1, 'intraday_std1': intraday_std1, 'aligned_output': aligned_output, 'spread_settle': spread_settle, 'data_last5_years': data_last5_years,'historical_sharp':historical_sharp}
def get_intraday_trend_signals(**kwargs): ticker = kwargs['ticker'] date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) breakout_method = 2 #print(ticker) ticker_head = cmi.get_contract_specs(ticker)['ticker_head'] contract_multiplier = cmi.contract_multiplier[ticker_head] ticker_class = cmi.ticker_class[ticker_head] daily_settles = gfp.get_futures_price_preloaded(ticker=ticker) daily_settles = daily_settles[daily_settles['settle_date'] <= datetime_to] daily_settles['ewma10'] = pd.ewma(daily_settles['close_price'], span=10) daily_settles['ewma50'] = pd.ewma(daily_settles['close_price'], span=50) if daily_settles['ewma10'].iloc[-1] > daily_settles['ewma50'].iloc[-1]: long_term_trend = 1 else: long_term_trend = -1 date_list = [exp.doubledate_shift_bus_days(double_date=date_to,shift_in_days=1)] date_list.append(date_to) intraday_data = opUtil.get_aligned_futures_data_intraday(contract_list=[ticker], date_list=date_list) intraday_data['time_stamp'] = [x.to_datetime() for x in intraday_data.index] intraday_data['hour_minute'] = [100*x.hour+x.minute for x in intraday_data['time_stamp']] end_hour = cmi.last_trade_hour_minute[ticker_head] start_hour = cmi.first_trade_hour_minute[ticker_head] if ticker_class in ['Ag']: start_hour1 = dt.time(0, 45, 0, 0) end_hour1 = dt.time(7, 45, 0, 0) selection_indx = [x for x in range(len(intraday_data.index)) if ((intraday_data['time_stamp'].iloc[x].time() < end_hour1) and(intraday_data['time_stamp'].iloc[x].time() >= start_hour1)) or ((intraday_data['time_stamp'].iloc[x].time() < end_hour) and(intraday_data['time_stamp'].iloc[x].time() >= start_hour))] else: selection_indx = [x for x in range(len(intraday_data.index)) if (intraday_data.index[x].to_datetime().time() < end_hour) and(intraday_data.index[x].to_datetime().time() >= start_hour)] selected_data = intraday_data.iloc[selection_indx] selected_data['mid_p'] = (selected_data['c1']['best_bid_p']+selected_data['c1']['best_ask_p'])/2 selected_data['ewma100'] = pd.ewma(selected_data['mid_p'], span=100) selected_data['ewma25'] = pd.ewma(selected_data['mid_p'], span=25) selected_data.reset_index(inplace=True,drop=True) datetime_to = cu.convert_doubledate_2datetime(date_to) range_start = dt.datetime.combine(datetime_to,dt.time(8,30,0,0)) range_end = dt.datetime.combine(datetime_to,dt.time(9,0,0,0)) first_30_minutes = selected_data[(selected_data['time_stamp'] >= range_start)& (selected_data['time_stamp'] <= range_end)] trading_data = selected_data[selected_data['time_stamp'] > range_end] trading_data_shifted = trading_data.shift(5) range_min = first_30_minutes['mid_p'].min() range_max = first_30_minutes['mid_p'].max() initial_range = range_max-range_min if breakout_method == 1: bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)& (trading_data['mid_p'] < range_max+0.5*initial_range)& (trading_data_shifted['mid_p']<range_max)& (trading_data['ewma25'] > range_max)& (trading_data['mid_p'] > trading_data['ewma100'])] bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)& (trading_data['mid_p'] > range_min-0.5*initial_range)& (trading_data_shifted['mid_p']>range_min)& (trading_data['ewma25'] < range_min)& (trading_data['mid_p'] < trading_data['ewma100'])] elif breakout_method == 2: bullish_breakout = pd.DataFrame() bearish_breakout = pd.DataFrame() if long_term_trend > 0: bullish_breakout = trading_data[(trading_data['mid_p'] > range_max)& (trading_data_shifted['mid_p']<range_max)& (long_term_trend == 1)& (trading_data['mid_p'] > trading_data['ewma100'])] elif long_term_trend < 0: bearish_breakout = trading_data[(trading_data['mid_p'] < range_min)& (trading_data_shifted['mid_p']>range_min)& (long_term_trend == -1)& (trading_data['mid_p'] < trading_data['ewma100'])] bullish_cross = trading_data[trading_data['mid_p'] > trading_data['ewma100']] bearish_cross = trading_data[trading_data['mid_p'] < trading_data['ewma100']] end_of_day_price = trading_data['mid_p'].iloc[-1] end_of_day_time_stamp = trading_data['time_stamp'].iloc[-1] valid_bearish_breakoutQ = False valid_bullish_breakoutQ = False bearish_breakout_price_entry = np.NaN bullish_breakout_price_entry = np.NaN if not bearish_breakout.empty: if bearish_breakout.index[0]+1 < trading_data.index[-1]: if trading_data['mid_p'].loc[bearish_breakout.index[0]+1]>range_min-0.5*initial_range: valid_bearish_breakoutQ = True bearish_breakout_price_entry = trading_data['mid_p'].loc[bearish_breakout.index[0]+1] bearish_breakout_time_stamp = trading_data['time_stamp'].loc[bearish_breakout.index[0]+1] if not bullish_breakout.empty: if bullish_breakout.index[0]+1<trading_data.index[-1]: if trading_data['mid_p'].loc[bullish_breakout.index[0]+1]<range_max+0.5*initial_range: valid_bullish_breakoutQ = True bullish_breakout_price_entry = trading_data['mid_p'].loc[bullish_breakout.index[0]+1] bullish_breakout_time_stamp = trading_data['time_stamp'].loc[bullish_breakout.index[0]+1] stop_loss = (range_max-range_min)*contract_multiplier pnl_list = [] direction_list = [] entry_time_list = [] exit_time_list = [] entry_price_list = [] exit_price_list = [] daily_trade_no_list = [] exit_type_list = [] ticker_list = [] if valid_bearish_breakoutQ: direction_list.append(-1) entry_time_list.append(bearish_breakout_time_stamp) entry_price_list.append(bearish_breakout_price_entry) daily_pnl = bearish_breakout_price_entry-end_of_day_price exit_price = end_of_day_price exit_type = 'eod' exit_time = end_of_day_time_stamp daily_trade_no = 1 #bullish_cross_stop_frame = bullish_cross[(bullish_cross['time_stamp'] > bearish_breakout_time_stamp)] #if (not bullish_cross_stop_frame.empty) and (bullish_cross_stop_frame.index[0]+1<trading_data.index[-1]): # daily_pnl = bearish_breakout_price_entry-trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1] # exit_price = trading_data['mid_p'].loc[bullish_cross_stop_frame.index[0]+1] # exit_type = 'oso' # exit_time = trading_data['time_stamp'].loc[bullish_cross_stop_frame.index[0]+1] #if valid_bullish_breakoutQ: # if bullish_breakout_time_stamp>bearish_breakout_time_stamp: # if bullish_breakout_time_stamp<exit_time: # daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry # exit_price = bullish_breakout_price_entry # exit_type = 'fso' # exit_time = bullish_breakout_time_stamp # else: # daily_trade_no = 2 exit_time_list.append(exit_time) exit_type_list.append(exit_type) daily_trade_no_list.append(daily_trade_no) pnl_list.append(daily_pnl) exit_price_list.append(exit_price) ticker_list.append(ticker) if valid_bullish_breakoutQ: direction_list.append(1) entry_time_list.append(bullish_breakout_time_stamp) entry_price_list.append(bullish_breakout_price_entry) daily_pnl = end_of_day_price-bullish_breakout_price_entry exit_price = end_of_day_price exit_type = 'eod' exit_time = end_of_day_time_stamp daily_trade_no = 1 bearish_cross_stop_frame = bearish_cross[(bearish_cross['time_stamp'] > bullish_breakout_time_stamp)] #if (not bearish_cross_stop_frame.empty) and (bearish_cross_stop_frame.index[0]+1 < trading_data.index[-1]): # daily_pnl = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1]-bullish_breakout_price_entry # exit_price = trading_data['mid_p'].loc[bearish_cross_stop_frame.index[0]+1] # exit_type = 'oso' # exit_time = trading_data['time_stamp'].loc[bearish_cross_stop_frame.index[0]+1] #if valid_bearish_breakoutQ: # if bearish_breakout_time_stamp>bullish_breakout_time_stamp: # if bearish_breakout_time_stamp<exit_time: # daily_pnl = bearish_breakout_price_entry-bullish_breakout_price_entry # exit_price = bearish_breakout_price_entry # exit_type = 'fso' # exit_time = bearish_breakout_time_stamp # else: # daily_trade_no = 2 exit_time_list.append(exit_time) exit_type_list.append(exit_type) daily_trade_no_list.append(daily_trade_no) pnl_list.append(daily_pnl) exit_price_list.append(exit_price) ticker_list.append(ticker) pnl_frame = pd.DataFrame.from_items([('ticker', ticker_list), ('ticker_head',ticker_head), ('direction', direction_list), ('entry_price', entry_price_list), ('exit_price', exit_price_list), ('pnl', pnl_list), ('entry_time', entry_time_list), ('exit_time', exit_time_list), ('exit_type', exit_type_list), ('daily_trade_no', daily_trade_no_list)]) pnl_frame['pnl'] = pnl_frame['pnl']*contract_multiplier return {'intraday_data': selected_data, 'range_min': range_min, 'range_max':range_max,'pnl_frame':pnl_frame,'stop_loss':stop_loss}
def get_simple_rate(**kwargs): as_of_date = kwargs["as_of_date"] date_to = kwargs["date_to"] if "date_from" in kwargs.keys(): date_from = kwargs["date_from"] else: date_from = as_of_date if "ticker_head" in kwargs.keys(): ticker_head = kwargs["ticker_head"] else: ticker_head = "ED" ta_output_dir = dn.get_dated_directory_extension(folder_date=as_of_date, ext="ta") file_name = ta_output_dir + "/" + ticker_head + "_interest_curve.pkl" # print('as_of_date: ' + str(as_of_date) + ', date_to: ' + str(date_to)) if os.path.isfile(file_name): price_frame = pd.read_pickle(file_name) if (not os.path.isfile(file_name)) or price_frame.empty: price_frame = gfp.get_futures_price_preloaded(ticker_head=ticker_head, settle_date=as_of_date) price_frame = price_frame[price_frame["close_price"].notnull()] price_frame.sort("tr_dte", ascending=True, inplace=True) price_frame["exp_date"] = [exp.get_futures_expiration(x) for x in price_frame["ticker"]] price_frame["implied_rate"] = 100 - price_frame["close_price"] price_frame.to_pickle(file_name) if price_frame.empty: return { "rate_output": np.NaN, "price_frame": pd.DataFrame(columns=["ticker", "cal_dte", "exp_date", "implied_rate"]), } datetime_to = cu.convert_doubledate_2datetime(date_to) datetime_from = cu.convert_doubledate_2datetime(date_from) price_frame_first = price_frame[price_frame["exp_date"] <= datetime_from] price_frame_middle = price_frame[ (price_frame["exp_date"] > datetime_from) & (price_frame["exp_date"] < datetime_to) ] if price_frame_middle.empty: if not price_frame_first.empty: rate_output = price_frame_first["implied_rate"].iloc[-1] / 100 else: rate_output = price_frame["implied_rate"].iloc[0] / 100 return { "rate_output": rate_output, "price_frame": price_frame[["ticker", "cal_dte", "exp_date", "implied_rate"]], } if price_frame_first.empty: first_rate = price_frame_middle["implied_rate"].iloc[0] first_period = (price_frame_middle["exp_date"].iloc[0].to_datetime() - datetime_from).days else: first_rate = price_frame_first["implied_rate"].iloc[-1] first_period = (price_frame_middle["exp_date"].iloc[0].to_datetime() - datetime_from).days last_rate = price_frame_middle["implied_rate"].iloc[-1] last_period = (datetime_to - price_frame_middle["exp_date"].iloc[-1].to_datetime()).days middle_discount = [ 1 + ( price_frame_middle["implied_rate"].iloc[x] * (price_frame_middle["cal_dte"].iloc[x + 1] - price_frame_middle["cal_dte"].iloc[x]) / 36500 ) for x in range(len(price_frame_middle.index) - 1) ] total_discount = ( np.prod(np.array(middle_discount)) * (1 + (first_rate * first_period / 36500)) * (1 + (last_rate * last_period / 36500)) ) total_period = ( (price_frame_middle["cal_dte"].iloc[-1] - price_frame_middle["cal_dte"].iloc[0]) + first_period + last_period ) rate_output = (total_discount - 1) * 365 / total_period return {"rate_output": rate_output, "price_frame": price_frame[["ticker", "cal_dte", "exp_date", "implied_rate"]]}
def update_futures_price_database_from_cme_file(**kwargs): ticker_head_list = cmi.cme_futures_tickerhead_list import time con = msu.get_my_sql_connection(**kwargs) if 'settle_date' in kwargs.keys(): run_date = kwargs['settle_date'] else: run_date = int(time.strftime('%Y%m%d')) #run_date = 20160225 data_vendor_id = 2 now = datetime.datetime.now() run_datetime = cu.convert_doubledate_2datetime(run_date) for ticker_head in ticker_head_list: #print(ticker_head) contract_list = [] bday_us = CustomBusinessDay(calendar=exp.get_calendar_4ticker_head(ticker_head)) if not exp.is_business_day(double_date=run_date, reference_tickerhead=ticker_head): continue cme_output = pcf.process_cme_futures_4tickerhead(ticker_head=ticker_head, report_date=run_date) settle_frame = cme_output['settle_frame'] for ticker_month in cmi.futures_contract_months[ticker_head]: ticker_month_num = cmi.letter_month_string.find(ticker_month)+1 max_cal_dte = cmi.get_max_cal_dte(ticker_head=ticker_head, ticker_month=ticker_month_num) contract_list.extend(cl.get_db_contract_list_filtered(expiration_date_from=run_date, expiration_date_to=cu.doubledate_shift(run_date, -max_cal_dte), ticker_head=ticker_head, ticker_month=ticker_month_num, con=con, instrument='futures')) contract_frame = pd.DataFrame(contract_list, columns=['symbol_id', 'ticker', 'expiration_date']) merged_frame = pd.merge(contract_frame,settle_frame, how='inner', on='ticker') merged_frame.sort('expiration_date', ascending=True, inplace=True) column_names = merged_frame.columns.tolist() symbol_id_indx = column_names.index('symbol_id') ticker_month_indx = column_names.index('ticker_month') open_indx = column_names.index('open') high_indx = column_names.index('high') low_indx = column_names.index('low') settle_indx = column_names.index('settle') volume_indx = column_names.index('volume') interest_indx = column_names.index('interest') expiration_indx = column_names.index('expiration_date') dts = pd.date_range(start=run_datetime, end=merged_frame['expiration_date'].iloc[-1], freq=bday_us) tuples = [tuple([data_vendor_id, x[symbol_id_indx], ticker_head, x[ticker_month_indx], run_datetime.date(), (x[expiration_indx]-run_datetime.date()).days, len([y for y in dts if y.to_datetime().date() < x[expiration_indx]]), now, now, None if np.isnan(x[open_indx]) else x[open_indx], None if np.isnan(x[high_indx]) else x[high_indx], None if np.isnan(x[low_indx]) else x[low_indx], None if np.isnan(x[settle_indx]) else x[settle_indx], None if np.isnan(x[volume_indx]) else x[volume_indx], None if np.isnan(x[interest_indx]) else x[interest_indx]]) for x in merged_frame.values] column_str = "data_vendor_id, symbol_id, ticker_head, ticker_month, price_date,cal_dte, tr_dte, created_date,last_updated_date, open_price, high_price, low_price, close_price, volume, open_interest" insert_str = ("%s, " * 15)[:-2] final_str = "REPLACE INTO daily_price (%s) VALUES (%s)" % (column_str, insert_str) msu.sql_execute_many_wrapper(final_str=final_str, tuples=tuples, con=con) if 'con' not in kwargs.keys(): con.close()
def get_vcs_signals_legacy(**kwargs): aligned_indicators_output = get_aligned_option_indicators_legacy(**kwargs) if not aligned_indicators_output['success']: return {'atm_vol_ratio': np.NaN, 'q': np.NaN, 'q2': np.NaN, 'q1': np.NaN, 'q5': np.NaN, 'fwd_vol_q': np.NaN, 'fwd_vol_q2': np.NaN, 'fwd_vol_q1': np.NaN, 'fwd_vol_q5': np.NaN, 'atm_real_vol_ratio': np.NaN, 'q_atm_real_vol_ratio': np.NaN, 'atm_real_vol_ratio_ratio': np.NaN, 'q_atm_real_vol_ratio_ratio': np.NaN, 'tr_dte_diff_percent': np.NaN,'downside': np.NaN, 'upside': np.NaN, 'theta1': np.NaN, 'theta2': np.NaN, 'hist': []} hist = aligned_indicators_output['hist'] current = aligned_indicators_output['current'] settle_datetime = cu.convert_doubledate_2datetime(kwargs['settle_date']) settle_datetime_1year_back = settle_datetime-dt.timedelta(360) settle_datetime_5year_back = settle_datetime-dt.timedelta(5*360) hist['atm_vol_ratio'] = hist['c1']['imp_vol']/hist['c2']['imp_vol'] fwd_var = hist['c2']['cal_dte']*(hist['c2']['imp_vol']**2)-hist['c1']['cal_dte']*(hist['c1']['imp_vol']**2) fwd_vol_sq = fwd_var/(hist['c2']['cal_dte']-hist['c1']['cal_dte']) fwd_vol_adj = np.sign(fwd_vol_sq)*((abs(fwd_vol_sq)).apply(np.sqrt)) hist['fwd_vol_adj'] = fwd_vol_adj fwd_var = current['cal_dte'][1]*(current['imp_vol'][1]**2)-current['cal_dte'][0]*(current['imp_vol'][0]**2) fwd_vol_sq = fwd_var/(current['cal_dte'][1]-current['cal_dte'][0]) fwd_vol_adj = np.sign(fwd_vol_sq)*(np.sqrt(abs(fwd_vol_sq))) atm_vol_ratio = current['imp_vol'][0]/current['imp_vol'][1] hist['atm_real_vol_ratio'] = hist['c1']['imp_vol']/hist['c1']['close2close_vol20'] atm_real_vol_ratio = current['imp_vol'][0]/current['close2close_vol20'][0] hist['atm_real_vol_ratio_ratio'] = (hist['c1']['imp_vol']/hist['c1']['close2close_vol20'])/(hist['c2']['imp_vol']/hist['c2']['close2close_vol20']) atm_real_vol_ratio_ratio = (current['imp_vol'][0]/current['close2close_vol20'][0])/(current['imp_vol'][0]/current['close2close_vol20'][0]) hist_1year = hist[hist.index >= settle_datetime_1year_back] hist_5year = hist[hist.index >= settle_datetime_5year_back] q = stats.get_quantile_from_number({'x': atm_vol_ratio, 'y': hist['atm_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))}) q2 = stats.get_quantile_from_number({'x': atm_vol_ratio, 'y': hist['atm_vol_ratio'].values[-40:], 'clean_num_obs': 30}) q1 = stats.get_quantile_from_number({'x': atm_vol_ratio, 'y': hist_1year['atm_vol_ratio'].values, 'clean_num_obs': max(50, round(3*len(hist_1year.index)/4))}) q5 = stats.get_quantile_from_number({'x': atm_vol_ratio, 'y': hist_5year['atm_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist_5year.index)/4))}) fwd_vol_q = stats.get_quantile_from_number({'x': fwd_vol_adj, 'y': hist['fwd_vol_adj'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))}) fwd_vol_q2 = stats.get_quantile_from_number({'x': fwd_vol_adj, 'y': hist['fwd_vol_adj'].values[-40:], 'clean_num_obs': 30}) fwd_vol_q1 = stats.get_quantile_from_number({'x': fwd_vol_adj, 'y': hist_1year['fwd_vol_adj'].values, 'clean_num_obs': max(50, round(3*len(hist_1year.index)/4))}) fwd_vol_q5 = stats.get_quantile_from_number({'x': fwd_vol_adj, 'y': hist_5year['fwd_vol_adj'].values, 'clean_num_obs': max(100, round(3*len(hist_5year.index)/4))}) q_atm_real_vol_ratio = stats.get_quantile_from_number({'x': atm_real_vol_ratio, 'y': hist['atm_real_vol_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))}) q_atm_real_vol_ratio_ratio = stats.get_quantile_from_number({'x': atm_real_vol_ratio_ratio, 'y': hist['atm_real_vol_ratio_ratio'].values, 'clean_num_obs': max(100, round(3*len(hist.index)/4))}) tr_dte_diff_percent = round(100*(current['tr_dte'][1]-current['tr_dte'][0])/current['tr_dte'][0]) profit5 = hist['c1']['profit5']-hist['c2']['profit5'] clean_indx = profit5.notnull() clean_data = hist[clean_indx] if clean_data.empty: downside = np.NaN upside = np.NaN else: last_available_align_date = clean_data.index[-1] clean_data = clean_data[clean_data.index >= last_available_align_date-dt.timedelta(5*365)] profit5 = clean_data['c1']['profit5']-clean_data['c2']['profit5'] percentile_vector = stats.get_number_from_quantile(y=profit5.values, quantile_list=[1, 15, 85, 99], clean_num_obs=max(100, round(3*len(profit5.values)/4))) downside = (percentile_vector[0]+percentile_vector[1])/2 upside = (percentile_vector[2]+percentile_vector[3])/2 return {'atm_vol_ratio': atm_vol_ratio, 'q': q, 'q2': q2, 'q1': q1, 'q5': q5, 'fwd_vol_q': fwd_vol_q, 'fwd_vol_q2': fwd_vol_q2, 'fwd_vol_q1': fwd_vol_q1, 'fwd_vol_q5': fwd_vol_q5, 'atm_real_vol_ratio': atm_real_vol_ratio, 'q_atm_real_vol_ratio': q_atm_real_vol_ratio, 'atm_real_vol_ratio_ratio': atm_real_vol_ratio_ratio, 'q_atm_real_vol_ratio_ratio': q_atm_real_vol_ratio_ratio, 'tr_dte_diff_percent': tr_dte_diff_percent, 'downside': downside, 'upside': upside, 'theta1': current['theta'][0], 'theta2': current['theta'][1], 'hist': hist}