def load_latest_tt_fills(**kwargs): file_list = os.listdir(dna.tt_fill_directory) num_files = len(file_list) time_list = [] for i in range(num_files): time_list.append(os.path.getmtime(dna.tt_fill_directory + '/' + file_list[i])) loc_latest_file = time_list.index(max(time_list)) dna.get_directory_name(ext='daily') + '/ttFills.xlsx' tt_export_frame = pd.read_excel(dna.get_directory_name(ext='daily') + '/ttFills.xlsx',header=None, names=['trade_date','trade_time','exchange','product','contract','prod_type','B/S','qty','price','P/F', 'route','account','order_tag', 'originator','current_user','tt_order_id','parent_id']) tt_export_frame_filtered = tt_export_frame[tt_export_frame['prod_type']=='Future'] if 'tags2exclude' in kwargs.keys(): tt_export_frame_filtered['order_tag'] = tt_export_frame_filtered['order_tag'].astype('str') tt_export_frame_filtered = tt_export_frame_filtered[[not any([y in x for y in kwargs['tags2exclude']]) for x in tt_export_frame_filtered['order_tag']]] return tt_export_frame_filtered
def assign_trades_2strategies(**kwargs): trade_source = kwargs['trade_source'] if trade_source == 'tt': formatted_fills = get_formatted_tt_fills(**kwargs) elif trade_source == 'cme_direct': formatted_fills = get_formatted_cme_direct_fills() elif trade_source == 'manual_entry': formatted_fills = get_formatted_manual_entry_fills() aggregate_trades = formatted_fills['aggregate_trades'] allocation_frame = pd.read_excel(dna.get_directory_name(ext='daily') + '/' + 'trade_allocation.xlsx') combined_list = [None]*len(allocation_frame.index) for i in range(len(allocation_frame.index)): if allocation_frame['criteria'][i]=='tickerhead': selected_trades = aggregate_trades[aggregate_trades['ticker_head'] == allocation_frame['value'][i]] elif allocation_frame['criteria'][i]=='ticker': selected_trades = aggregate_trades[aggregate_trades['ticker'] == allocation_frame['value'][i]] combined_list[i] = selected_trades[['ticker','option_type','strike_price','trade_price','trade_quantity','instrument','real_tradeQ']] combined_list[i]['alias'] = allocation_frame['alias'][i] return pd.concat(combined_list).reset_index(drop=True)
def send_hrsn_report(**kwargs): daily_dir = dna.get_directory_name(ext='daily') if 'report_date' in kwargs.keys(): report_date = kwargs['report_date'] else: report_date = exp.doubledate_shift_bus_days() ibo_dir = ts.create_strategy_output_dir(strategy_class='os', report_date=report_date) cov_data_integrity = '' try: with open(ibo_dir + '/' + 'covDataIntegrity.txt','r') as text_file: cov_data_integrity = text_file.read() except Exception: pass try: expiration_report = ef.get_expiration_report(report_date=report_date, con=kwargs['con']) expiration_report = expiration_report[expiration_report['tr_days_2roll'] < 5] if expiration_report.empty: expiration_text = 'No near expirations.' else: expiration_text = 'Check for approaching expirations!' except Exception: expiration_text = 'Check expiration report for errors!' se.send_email_with_attachment(subject='hrsn_' + str(report_date), email_text='cov_data_integrity: ' + cov_data_integrity + "\r\n" + expiration_text, attachment_list = [daily_dir + '/' + 'pnl_final_' + str(report_date) + '.xlsx', daily_dir + '/' + 'followup_' + str(report_date) + '.xlsx'])
def send_wh_report(**kwargs): if 'report_date' in kwargs.keys(): report_date = kwargs['report_date'] else: report_date = exp.doubledate_shift_bus_days() ta_output_dir = dn.get_dated_directory_extension(folder_date=report_date, ext='ta') strategy_frame = tpm.get_daily_pnl_snapshot(as_of_date=report_date, name='final') total_pnl_row = strategy_frame[strategy_frame.alias == 'TOTAL'] report_date_str = cu.convert_datestring_format({'date_string': str(report_date), 'format_from': 'yyyymmdd', 'format_to': 'dd/mm/yyyy'}) config_output = su.read_config_file(file_name=dna.get_directory_name(ext='daily') + '/riskAndMargin.txt') email_text = "Expected Maximum Drawdown: " + config_output['emd'] + "K" + \ "\nMargin: " + str(int(config_output['iceMargin']) + int(config_output['cmeMargin'])) + "K" + \ "\nNet Liquidating Value: " + config_output['pnl'] + "K" + \ "\n \nSee attached for individual strategy pnls." se.send_email_with_attachment(send_from='*****@*****.**', send_to=['*****@*****.**','*****@*****.**'], sender_account_alias='wh_trading', subject='Daily PnL for ' + report_date_str + ' is: ' + '${:,}'.format(total_pnl_row['daily_pnl'].iloc[0]), email_text=email_text, attachment_list = [ta_output_dir + '/' + 'pnl_final.xlsx'], attachment_name_list=['PnLs.xlsx'])
def get_futures_price_preloaded(**kwargs): if 'ticker_head' in kwargs.keys(): ticker_head = kwargs['ticker_head'] else: ticker = kwargs['ticker'] contract_specs_output = cmi.get_contract_specs(ticker) ticker_head = contract_specs_output['ticker_head'] file_ticker = cmi.mini_contract_dictionary.get( ticker_head, ticker_head) + contract_specs_output['ticker_month_str'] + str( contract_specs_output['ticker_year']) if 'futures_data_dictionary' in kwargs.keys(): data_out = kwargs['futures_data_dictionary'][ticker_head] else: presaved_futures_data_folder = dna.get_directory_name( ext='presaved_futures_data') file_ticker_head = cmi.mini_contract_dictionary.get( ticker_head, ticker_head) if os.path.isfile(presaved_futures_data_folder + '/' + file_ticker_head + '.pkl'): data_out = pd.read_pickle(presaved_futures_data_folder + '/' + file_ticker_head + '.pkl') else: data_out = pd.DataFrame() return data_out if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] if isinstance(settle_date, int): data_out = data_out[data_out['settle_date'] == cu.convert_doubledate_2datetime(settle_date)] elif isinstance(settle_date, dt.datetime): data_out = data_out[data_out['settle_date'] == settle_date] if 'settle_date_from_exclusive' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] > cu.convert_doubledate_2datetime( kwargs['settle_date_from_exclusive'])] if 'settle_date_from' in kwargs.keys(): data_out = data_out[ data_out['settle_date'] >= cu.convert_doubledate_2datetime( kwargs['settle_date_from'])] if 'settle_date_to' in kwargs.keys(): settle_date_to = kwargs['settle_date_to'] if isinstance(settle_date_to, int): data_out = data_out[ data_out['settle_date'] <= cu.convert_doubledate_2datetime( kwargs['settle_date_to'])] elif isinstance(settle_date_to, dt.datetime): data_out = data_out[data_out['settle_date'] <= settle_date_to] if 'ticker' in kwargs.keys(): data_out = data_out[data_out['ticker'] == file_ticker] return data_out
def get_strategy_target_size(**kwargs): config_output = su.read_text_file( file_name=dna.get_directory_name(ext='c#config') + '/BetSize.txt') bet_size = float(config_output[0]) return bet_size * target_size_multiplier_dictionary[ kwargs['strategy_class']]
def load_cme__fills(**kwargs): fill_frame = pd.read_csv(dna.get_directory_name(ext='daily') + '/' + cme_direct_fill_file_name, header=1) fill_frame_filtered = fill_frame[fill_frame['IsStrategy'] == False] fill_frame_filtered.reset_index(inplace=True,drop=True) return fill_frame_filtered[['ContractCode', 'Side', 'Price', 'FilledQuantity']]
def get_stock_price_preloaded(**kwargs): ticker = kwargs['ticker'] if 'data_source' in kwargs.keys(): data_source = kwargs['data_source'] else: data_source = 'iex' if data_source == 'iex': file_dir = dna.get_directory_name(ext='iex_stock_data') else: file_dir = dna.get_directory_name(ext='stock_data') if 'stock_data_dictionary' in kwargs.keys(): data_out = kwargs['stock_data_dictionary'][ticker] else: if not os.path.isfile(file_dir + '/' + ticker + '.pkl'): ssd.save_stock_data(symbol_list=[ticker],data_source=data_source) data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl') report_date = exp.doubledate_shift_bus_days() if cu.convert_doubledate_2datetime(report_date)>data_out['settle_datetime'].iloc[-1].to_pydatetime(): ssd.save_stock_data(symbol_list=[ticker],data_source=data_source) data_out = pd.read_pickle(file_dir + '/' + ticker + '.pkl') if 'settle_date' in kwargs.keys(): settle_date = kwargs['settle_date'] if isinstance(settle_date,int): data_out = data_out[data_out['settle_datetime'] == cu.convert_doubledate_2datetime(settle_date)] elif isinstance(settle_date,dt.datetime): data_out = data_out[data_out['settle_datetime'] == settle_date] if 'settle_date_from' in kwargs.keys(): data_out = data_out[data_out['settle_datetime']>=cu.convert_doubledate_2datetime(kwargs['settle_date_from'])] if 'settle_date_to' in kwargs.keys(): data_out = data_out[data_out['settle_datetime']<=cu.convert_doubledate_2datetime(kwargs['settle_date_to'])] return data_out
def get_pca_seasonality_adjustments(**kwargs): ticker_head = kwargs['ticker_head'] if 'file_date_to' in kwargs.keys(): file_date_to = kwargs['file_date_to'] else: file_date_to = 20160219 if 'years_back' in kwargs.keys(): years_back = kwargs['years_back'] else: years_back = 10 if 'date_to' in kwargs.keys(): date_to = kwargs['date_to'] else: date_to = file_date_to date5_years_ago = cu.doubledate_shift(date_to, 5 * 365) backtest_output_dir = dna.get_directory_name(ext='backtest_results') file_name = ticker_head + '_' + str(file_date_to) + '_' + str( years_back) + '_z' if os.path.isfile(backtest_output_dir + '/curve_pca/' + file_name + '.pkl'): backtest_results = pd.read_pickle(backtest_output_dir + '/curve_pca/' + file_name + '.pkl') else: return pd.DataFrame.from_items([('monthSpread', [1] * 12 + [6] * 2), ('ticker_month_front', list(range(1, 13)) + [6] + [12]), ('z_seasonal_mean', [0] * 14)]) entire_report = pd.concat(backtest_results['report_results_list']) selected_report = entire_report[ (entire_report['report_date'] <= date_to) & (entire_report['report_date'] >= date5_years_ago)] selected_report = selected_report[(selected_report['tr_dte_front'] > 80) & (selected_report['monthSpread'] < 12)] grouped = selected_report.groupby(['monthSpread', 'ticker_month_front']) seasonality_adjustment = pd.DataFrame() seasonality_adjustment['monthSpread'] = ( grouped['monthSpread'].first()).values seasonality_adjustment['ticker_month_front'] = ( grouped['ticker_month_front'].first()).values seasonality_adjustment['z_seasonal_mean'] = (grouped['z'].mean()).values return seasonality_adjustment
def get_daily_price_data4ticker(**kwargs): date_to = kwargs['date_to'] datetime_to = cu.convert_doubledate_2datetime(date_to) try: data_out = pd.read_pickle( dna.get_directory_name(ext='binance') + '/daily/' + kwargs['ticker'] + '.pkl') except: return pd.DataFrame() #data_out['pydate'] = [x.to_pydatetime().date() for x in data_out['openDatetime']] return data_out[data_out['openDate'] <= datetime_to.date()]
def get_strategy_class_historical_pnls(**kwargs): con = msu.get_my_sql_connection(**kwargs) if 'as_of_date' in kwargs.keys(): as_of_date = kwargs['as_of_date'] else: as_of_date = exp.doubledate_shift_bus_days() strategy_frame = ts.select_strategies(con=con, open_date_to=as_of_date) strategy_frame['strategy_class'] = [ sc.convert_from_string_to_dictionary( string_input=strategy_frame['description_string'][x]) ['strategy_class'] for x in range(len(strategy_frame.index)) ] unique_strategy_class_list = strategy_frame['strategy_class'].unique() time_series_list = [None] * len(unique_strategy_class_list) for i in range(len(unique_strategy_class_list)): strategy_frame_selected = strategy_frame[ strategy_frame['strategy_class'] == unique_strategy_class_list[i]] pnl_out = [ tpnl.get_strategy_pnl( alias=x, as_of_date=as_of_date, con=con)['pnl_frame'][['settle_date', 'total_pnl']] for x in strategy_frame_selected['alias'] ] [x.set_index('settle_date', drop=True, inplace=True) for x in pnl_out] time_series_list[i] = pd.concat(pnl_out, axis=1).fillna(0).sum(axis=1) if 'con' not in kwargs.keys(): con.close() merged_pnl = pd.concat(time_series_list, axis=1, keys=unique_strategy_class_list).fillna(0) merged_pnl['total'] = merged_pnl.sum(axis=1) output_dir = dna.get_directory_name(ext='daily') writer = pd.ExcelWriter(output_dir + '/historical_performance_' + str(as_of_date) + '.xlsx', engine='xlsxwriter') merged_pnl.to_excel(writer, sheet_name='timeSeries') writer.save() return merged_pnl
def get_formatted_manual_entry_fills(**kwargs): fill_frame = pd.read_csv(dna.get_directory_name(ext='daily') + '/' + manual_trade_entry_file_name) formatted_frame = fill_frame formatted_frame.rename(columns={'optionType': 'option_type', 'strikePrice': 'strike_price', 'tradePrice': 'trade_price', 'quantity': 'trade_quantity'}, inplace=True) formatted_frame['strike_price'] = formatted_frame['strike_price'].astype('float64') formatted_frame['PQ'] = formatted_frame['trade_price']*formatted_frame['trade_quantity'] formatted_frame['instrument'] = 'O' formatted_frame.loc[formatted_frame['option_type'].isnull(),'instrument'] = 'F' formatted_frame.loc[[cmi.is_stockQ(x) for x in formatted_frame['ticker']], 'instrument'] = 'S' option_type = formatted_frame['option_type'] formatted_frame['option_type']= option_type.where(pd.notnull(option_type),None) option_indx = formatted_frame['instrument'] == 'O' formatted_frame['generalized_ticker'] = formatted_frame['ticker'] formatted_frame['generalized_ticker'][option_indx] = formatted_frame['ticker'][option_indx] + '-' + \ formatted_frame['option_type'][option_indx] + '-' + \ formatted_frame['strike_price'][option_indx].astype(str) formatted_frame['side'] = np.sign(formatted_frame['trade_quantity']) formatted_frame['ticker_head'] = [cmi.get_contract_specs(x)['ticker_head'] for x in formatted_frame['ticker']] grouped = formatted_frame.groupby(['generalized_ticker', 'side']) aggregate_trades = pd.DataFrame() aggregate_trades['trade_price'] = grouped['PQ'].sum()/grouped['trade_quantity'].sum() aggregate_trades['trade_quantity'] = grouped['trade_quantity'].sum() aggregate_trades['ticker'] = grouped['ticker'].first() aggregate_trades['ticker_head'] = grouped['ticker_head'].first() aggregate_trades['instrument'] = grouped['instrument'].first() aggregate_trades['option_type'] = grouped['option_type'].first() aggregate_trades['strike_price'] = grouped['strike_price'].first() aggregate_trades['real_tradeQ'] = True return {'raw_trades': fill_frame, 'aggregate_trades': aggregate_trades }
def load_aligend_options_data_file(**kwargs): ticker_head = kwargs['ticker_head'] tr_dte_center = kwargs['tr_dte_center'] option_data_dir = dna.get_directory_name(ext='aligned_time_series_output') if 'delta_center' in kwargs.keys(): delta_center = kwargs['delta_center'] else: delta_center = 0.5 if 'model' in kwargs.keys(): model = kwargs['model'] else: model = 'BS' if 'contract_month_letter' in kwargs.keys(): contract_month_str = '_' + kwargs['contract_month_letter'] else: contract_month_str = '' if 'column_names' in kwargs.keys(): column_names = kwargs['column_names'] else: column_names = get_column_names_4option_data() file_dir = ticker_head + '_' + str( delta_center) + '_' + model + '_20_510204060_' + str( tr_dte_center) + contract_month_str + '.mat' if os.path.isfile(option_data_dir + '/' + file_dir): try: mat_output = scipy.io.loadmat(option_data_dir + '/' + file_dir) data_frame_out = pd.DataFrame(mat_output['alignedDataMatrix'], columns=column_names) except Exception: mat_output = h5py.File(option_data_dir + '/' + file_dir) data_frame_out = pd.DataFrame( mat_output['alignedDataMatrix'][()].transpose(), columns=column_names) else: data_frame_out = pd.DataFrame(columns=column_names) return data_frame_out
def get_dated_directory_extension(**kwargs): if 'folder_date' in kwargs.keys(): folder_date = kwargs['folder_date'] else: folder_date = exp.doubledate_shift_bus_days() if 'ext' not in kwargs.keys(): print('Need to provide a valid ext !') return directory_name = dna.get_directory_name(**kwargs) dated_directory_name = directory_name + '/' + cu.get_directory_extension( folder_date) if not os.path.exists(dated_directory_name): os.makedirs(dated_directory_name) return dated_directory_name
def create_strategy_output_dir(**kwargs): strategy_class = kwargs['strategy_class'] report_date = kwargs['report_date'] strategy_output_folder = dna.get_directory_name(ext='strategy_output') if strategy_class == 'futures_butterfly': output_dir = strategy_output_folder + '/futures_butterfly/' + cu.get_directory_extension( report_date) elif strategy_class == 'intraday_futures_experimental': output_dir = strategy_output_folder + '/ife/' + cu.get_directory_extension( report_date) else: output_dir = strategy_output_folder + '/' + strategy_class + '/' + cu.get_directory_extension( report_date) if not os.path.exists(output_dir): os.makedirs(output_dir) return output_dir
def save_ib_data(**kwargs): if 'duration_str' in kwargs.keys(): duration_str = kwargs['duration_str'] else: duration_str = '2 M' app = algo.Algo() con = msu.get_my_sql_connection() date_now = cu.get_doubledate() datetime_now = dt.datetime.now() report_date = exp.doubledate_shift_bus_days() ticker_head_list = cmi.cme_futures_tickerhead_list data_list = [ gfp.get_futures_price_preloaded(ticker_head=x, settle_date=report_date) for x in ticker_head_list ] ticker_frame = pd.concat(data_list) ticker_frame = ticker_frame[~((ticker_frame['ticker_head'] == 'ED') & (ticker_frame['tr_dte'] < 250))] ticker_frame = ticker_frame[~((ticker_frame['ticker_head'] == 'GC') | (ticker_frame['ticker_head'] == 'SI'))] ticker_frame.sort_values(['ticker_head', 'volume'], ascending=[True, False], inplace=True) ticker_frame.drop_duplicates(subset=['ticker_head'], keep='first', inplace=True) app.ticker_list = list(ticker_frame['ticker']) app.output_dir = dna.get_directory_name(ext='ib_data') app.durationStr = duration_str app.con = con app.connect(client_id=5) app.run()
def save_daily_price_data4ticker(**kwargs): if 'client' in kwargs.keys(): client = kwargs['client'] else: client = btu.get_binance_client() ticker = kwargs['ticker'] file_name = dna.get_directory_name( ext='binance') + '/daily/' + ticker + '.pkl' if os.path.isfile(file_name): old_frame = pd.read_pickle(file_name) new_frame = gbp.get_klines( ticker=ticker, interval='1d', start_str=(old_frame['openDatetime'].iloc[-1] - dt.timedelta(days=5)).strftime('%m/%d/%y')) new_frame['frame_indx'] = 1 old_frame['frame_indx'] = 0 merged_data = pd.concat([old_frame, new_frame], ignore_index=True) merged_data.sort_values(['openDate', 'frame_indx'], ascending=[True, False], inplace=True) merged_data.drop_duplicates(subset=['openDate'], keep='first', inplace=True) price_frame = merged_data.drop('frame_indx', 1, inplace=False) else: price_frame = gbp.get_klines(ticker=ticker, interval='1d', start_str='01/01/2017', client=client) price_frame.to_pickle(file_name)
def get_spreads_4date(**kwargs): futures_dataframe = cl.generate_futures_list_dataframe(**kwargs) if 'volume_filter' in kwargs.keys(): volume_filter = kwargs['volume_filter'] futures_dataframe = futures_dataframe[futures_dataframe['volume'] > volume_filter] futures_dataframe.reset_index(drop=True, inplace=True) futures_dataframe['yearMonth'] = 12*futures_dataframe['ticker_year']+futures_dataframe['ticker_month'] futures_dataframe['yearMonthMerge'] = futures_dataframe['yearMonth'] futures_dataframe = futures_dataframe[['ticker','yearMonth','yearMonthMerge','ticker_head','volume']] spread_frame = pd.read_excel(dna.get_directory_name(ext='python_file') + '/opportunity_constructs/user_defined_spreads.xlsx') output_frame = pd.DataFrame() for i in range(len(spread_frame.index)): tickerhead1 = spread_frame['tickerHead1'].iloc[i] tickerhead2 = spread_frame['tickerHead2'].iloc[i] tickerhead3 = spread_frame['tickerHead3'].iloc[i] tickerhead4 = spread_frame['tickerHead4'].iloc[i] frame1 = futures_dataframe[futures_dataframe['ticker_head'] == tickerhead1] frame2 = futures_dataframe[futures_dataframe['ticker_head'] == tickerhead2] frame3 = futures_dataframe[futures_dataframe['ticker_head'] == tickerhead3] frame4 = futures_dataframe[futures_dataframe['ticker_head'] == tickerhead4] merged11 = pd.merge(frame1, frame2, how='inner', on='yearMonthMerge') frame2['yearMonthMerge'] = frame2['yearMonth']+1 merged12 = pd.merge(frame1, frame2, how='inner', on='yearMonthMerge') frame2['yearMonthMerge'] = frame2['yearMonth']-1 merged10 = pd.merge(frame1, frame2, how='inner', on='yearMonthMerge') if frame3.empty: output_frame2 = pd.concat([merged11, merged12, merged10]) spread_i = pd.DataFrame() spread_i['contract1'] = output_frame2['ticker_x'] spread_i['contract2'] = output_frame2['ticker_y'] spread_i['contract3'] = None spread_i['ticker_head1'] = tickerhead1 spread_i['ticker_head2'] = tickerhead2 spread_i['ticker_head3'] = None spread_i['volume1'] = output_frame2['volume_x'] spread_i['volume2'] = output_frame2['volume_y'] spread_i['volume3'] = None output_frame = pd.concat([output_frame, spread_i]) elif frame4.empty: frame3 = futures_dataframe[futures_dataframe['ticker_head'] == tickerhead3] merged111 = pd.merge(merged11, frame3, how='inner', on='yearMonthMerge') merged121 = pd.merge(merged12, frame3, how='inner', on='yearMonthMerge') merged101 = pd.merge(merged10, frame3, how='inner', on='yearMonthMerge') frame3['yearMonthMerge'] = frame3['yearMonth']+1 merged112 = pd.merge(merged11, frame3, how='inner', on='yearMonthMerge') merged122 = pd.merge(merged12, frame3, how='inner', on='yearMonthMerge') frame3['yearMonthMerge'] = frame3['yearMonth']-1 merged110 = pd.merge(merged11, frame3, how='inner', on='yearMonthMerge') merged100 = pd.merge(merged10, frame3, how='inner', on='yearMonthMerge') output_frame3 = pd.concat([merged111,merged121,merged101, merged112,merged122,merged110,merged100],sort=True) spread_i = pd.DataFrame() spread_i['contract1'] = output_frame3['ticker_x'] spread_i['contract2'] = output_frame3['ticker_y'] spread_i['contract3'] = output_frame3['ticker'] spread_i['ticker_head1'] = tickerhead1 spread_i['ticker_head2'] = tickerhead2 spread_i['ticker_head3'] = tickerhead3 spread_i['volume1'] = output_frame3['volume_x'] spread_i['volume2'] = output_frame3['volume_y'] spread_i['volume3'] = output_frame3['volume'] output_frame = pd.concat([output_frame,spread_i]) output_frame['spread_description'] = output_frame.apply(lambda x: x['ticker_head1']+ '_' +x['ticker_head2'] if x['ticker_head3'] is None else x['ticker_head1']+ '_' +x['ticker_head2'] + '_' + x['ticker_head3'] , axis=1) output_frame['min_volume'] = output_frame.apply(lambda x: min(x['volume1'],x['volume2']) if x['ticker_head3'] is None else min(x['volume1'],x['volume2'],x['volume3']),axis=1) output_frame.sort_values(['spread_description','min_volume'],ascending=[True, False],inplace=True) output_frame.drop_duplicates('spread_description',inplace=True) output_frame.reset_index(drop=True,inplace=True) return output_frame
def get_cme_direct_prices(**kwargs): cme_frame = pd.read_csv( dna.get_directory_name(ext='daily') + '/' + price_file_name) return cme_frame
import quandl_data.get_data_quandl as gdq import shared.directory_names_aux as dna import contract_utilities.contract_meta_info as cmi import shared.calendar_utilities as cu import datetime as dt import os.path import pandas as pd presaved_cot_data_folder = dna.get_directory_name( ext='commitments_of_traders_data') db_2_quandl_dictionary = { 'GC': '088691', 'SI': '084691', 'EC': '099741', 'BP': '096742', 'JY': '097741', 'AD': '232741', 'CD': '090741', 'TU': '042601', 'FV': '044601', 'TY': '043602', 'US': '020601', 'ED': '132741', 'ES': '13874A', 'NQ': '209742', 'CL': '067651', 'HO': '022651', 'RB': '111659', 'NG': '023651', 'C': '002602',
def main(): app = algo.Algo() report_date = exp.doubledate_shift_bus_days() todays_date = cu.get_doubledate() con = msu.get_my_sql_connection() vcs_output = vcs.generate_vcs_sheet_4date(date_to=report_date) vcs_pairs = vcs_output['vcs_pairs'] filter_out = of.get_vcs_filters(data_frame_input=vcs_pairs, filter_list=['long2', 'short2']) vcs_pairs = filter_out['selected_frame'] vcs_pairs = vcs_pairs[vcs_pairs['downside'].notnull() & vcs_pairs['upside'].notnull()] # &(vcs_pairs.tickerClass!='Energy') vcs_pairs = vcs_pairs[(vcs_pairs['trDte1'] >= 50) & (vcs_pairs.tickerClass != 'Metal') & (vcs_pairs.tickerClass != 'FX') & (vcs_pairs.tickerClass != 'Energy')] vcs_pairs = vcs_pairs[((vcs_pairs['Q'] <= 30) & (vcs_pairs['fwdVolQ'] >= 30)) | ((vcs_pairs['Q'] >= 70) & (vcs_pairs['fwdVolQ'] <= 70))] vcs_pairs.reset_index(drop=True, inplace=True) vcs_pairs['underlying_ticker1'] = [ omu.get_option_underlying(ticker=x) for x in vcs_pairs['ticker1'] ] vcs_pairs['underlying_ticker2'] = [ omu.get_option_underlying(ticker=x) for x in vcs_pairs['ticker2'] ] vcs_pairs['underlying_tickerhead'] = [ cmi.get_contract_specs(x)['ticker_head'] for x in vcs_pairs['underlying_ticker1'] ] futures_data_dictionary = { x: gfp.get_futures_price_preloaded(ticker_head=x) for x in vcs_pairs['underlying_tickerhead'].unique() } proxy_output_list1 = [ up.get_underlying_proxy_ticker( ticker=x, settle_date=report_date, futures_data_dictionary=futures_data_dictionary) for x in vcs_pairs['underlying_ticker1'] ] vcs_pairs['proxy_ticker1'] = [x['ticker'] for x in proxy_output_list1] vcs_pairs['add_2_proxy1'] = [x['add_2_proxy'] for x in proxy_output_list1] proxy_output_list2 = [ up.get_underlying_proxy_ticker( ticker=x, settle_date=report_date, futures_data_dictionary=futures_data_dictionary) for x in vcs_pairs['underlying_ticker2'] ] vcs_pairs['proxy_ticker2'] = [x['ticker'] for x in proxy_output_list2] vcs_pairs['add_2_proxy2'] = [x['add_2_proxy'] for x in proxy_output_list2] vcs_pairs['expiration_date1'] = [ int( exp.get_expiration_from_db(instrument='options', ticker=x, con=con).strftime('%Y%m%d')) for x in vcs_pairs['ticker1'] ] vcs_pairs['expiration_date2'] = [ int( exp.get_expiration_from_db(instrument='options', ticker=x, con=con).strftime('%Y%m%d')) for x in vcs_pairs['ticker2'] ] vcs_pairs['interest_date1'] = [ grfs.get_simple_rate(as_of_date=report_date, date_to=x)['rate_output'] for x in vcs_pairs['expiration_date1'] ] vcs_pairs['interest_date2'] = [ grfs.get_simple_rate(as_of_date=report_date, date_to=x)['rate_output'] for x in vcs_pairs['expiration_date2'] ] vcs_pairs['exercise_type'] = [ cmi.get_option_exercise_type(ticker_head=x) for x in vcs_pairs['tickerHead'] ] admin_dir = dna.get_directory_name(ext='admin') risk_file_out = su.read_text_file(file_name=admin_dir + '/RiskParameter.txt') vcs_risk_parameter = 5 * 2 * float(risk_file_out[0]) vcs_pairs['long_quantity'] = vcs_risk_parameter / abs( vcs_pairs['downside']) vcs_pairs['short_quantity'] = vcs_risk_parameter / vcs_pairs['upside'] vcs_pairs['long_quantity'] = vcs_pairs['long_quantity'].round() vcs_pairs['short_quantity'] = vcs_pairs['short_quantity'].round() vcs_pairs['alias'] = [ generate_vcs_alias(vcs_row=vcs_pairs.iloc[x]) for x in range(len(vcs_pairs.index)) ] vcs_pairs['call_mid_price1'] = np.nan vcs_pairs['put_mid_price1'] = np.nan vcs_pairs['call_mid_price2'] = np.nan vcs_pairs['put_mid_price2'] = np.nan vcs_pairs['call_iv1'] = np.nan vcs_pairs['put_iv1'] = np.nan vcs_pairs['call_iv2'] = np.nan vcs_pairs['put_iv2'] = np.nan vcs_pairs['underlying_mid_price1'] = np.nan vcs_pairs['underlying_mid_price2'] = np.nan vcs_pairs['proxy_mid_price1'] = np.nan vcs_pairs['proxy_mid_price2'] = np.nan vcs_pairs['current_strike1'] = np.nan vcs_pairs['current_strike2'] = np.nan ta_folder = dn.get_dated_directory_extension(folder_date=todays_date, ext='ta') app.vcs_pairs = vcs_pairs app.con = con app.futures_data_dictionary = futures_data_dictionary app.report_date = report_date app.todays_date = todays_date app.log = lg.get_logger(file_identifier='vcs', log_level='INFO') app.trade_file = ta_folder + '/trade_dir.csv' app.vcs_risk_parameter = vcs_risk_parameter app.connect(client_id=3) app.run()
import contract_utilities.expiration as exp import ta.strategy as ts import shutil as sutil import shared.directory_names_aux as dna import shared.directory_names as dn import formats.utils as futil daily_dir = dna.get_directory_name(ext='daily') def prepare_strategy_daily(**kwargs): strategy_class = kwargs['strategy_class'] if 'report_date' in kwargs.keys(): report_date = kwargs['report_date'] else: report_date = exp.doubledate_shift_bus_days() output_dir = ts.create_strategy_output_dir(strategy_class=strategy_class, report_date=report_date) sutil.copyfile( output_dir + '/' + futil.xls_file_names[strategy_class] + '.xlsx', daily_dir + '/' + futil.xls_file_names[strategy_class] + '_' + str(report_date) + '.xlsx') def move_from_dated_folder_2daily_folder(**kwargs): ext = kwargs['ext']
def load_and_convert_man_position_file(**kwargs): positions_directory = dna.get_directory_name(ext='man_positions') file_list = os.listdir(positions_directory) num_files = len(file_list) time_list = [] for i in range(num_files): time_list.append( os.path.getmtime(positions_directory + '/' + file_list[i])) loc_latest_file = time_list.index(max(time_list)) man_frame = pd.read_csv(positions_directory + '/' + file_list[loc_latest_file]) man_frame.rename(columns={ 'Strike': 'strike_price', 'Option Type': 'option_type', 'Net Qty': 'qty', 'Futures Code': 'Instrument', 'Contract Maturity': 'Prompt' }, inplace=True) man_frame['ticker_head'] = [ conversion_from_man_ticker_head[x] for x in man_frame['Instrument'] ] man_frame['strike_multiplier'] = [ man_strike_multiplier.get(x, 1) for x in man_frame['ticker_head'] ] man_frame['ticker'] = [ man_frame['ticker_head'].iloc[x] + cmi.full_letter_month_list[int(man_frame['Prompt'].iloc[x] % 100) - 1] + str(m.floor(man_frame['Prompt'].iloc[x] / 100)) for x in range(len(man_frame.index)) ] man_frame.rename(columns={ 'Strike': 'strike_price', 'OptionType': 'option_type', 'NetQty': 'qty' }, inplace=True) man_frame['strike_price'] = round( man_frame['strike_multiplier'] * man_frame['strike_price'], 4) man_frame['instrumet'] = 'F' option_indx = (man_frame['option_type'] == 'C') | (man_frame['option_type'] == 'P') man_frame['instrumet'][option_indx] = 'O' man_frame['generalized_ticker'] = man_frame['ticker'] man_frame['generalized_ticker'][option_indx] = man_frame['ticker'][option_indx] + '-' + \ man_frame['option_type'][option_indx] + '-' + \ man_frame['strike_price'][option_indx].astype(str) man_frame = man_frame[man_frame['qty'] != 0] man_frame['generalized_ticker'] = [ x.rstrip('0').rstrip('.') for x in man_frame['generalized_ticker'] ] return man_frame[['generalized_ticker', 'qty']]
def load_and_convert_wh_position_file(**kwargs): positions_directory = dna.get_directory_name(ext='wh_positions') file_name = 'open pos .tul01 5-1.xlsx' double_date = cu.get_doubledate() century_mark = m.floor(double_date / 1e6) * 100 wh_frame = pd.read_excel(positions_directory + '/' + file_name, header=None, names=['raw_symbol', 'qty']) raw_symbol_list = [] qty_list = [] instrument_list = [] current_instrument = '' direction_list = [] current_direction = '' strike_price_list = [] option_type_list = [] ticker_list = [] ticker_head_list = [] for i in range(len(wh_frame.index) - 1): if len(str(wh_frame['raw_symbol'].iloc[i])) > 4 and (str( wh_frame['qty'].iloc[i]).isnumeric()): raw_symbol_list.append(wh_frame['raw_symbol'].iloc[i]) instrument_list.append(current_instrument) split_out = wh_frame['raw_symbol'].iloc[i].split(' ') if split_out[0] in ['CALL', 'PUT']: strike_price_list.append(float(split_out[-1])) option_type_list.append(split_out[0][0]) if split_out[0] == 'CALL': month_indx = 1 year_indx = 2 else: month_indx = 2 year_indx = 3 else: strike_price_list.append(np.nan) option_type_list.append(None) month_indx = 0 year_indx = 1 ticker_head = conversion_from_man_ticker_head[current_instrument] ticker_head_list.append(ticker_head) ticker_list.append(ticker_head + cmi.full_letter_month_list[ cu.three_letter_month_dictionary[split_out[month_indx]] - 1] + str(century_mark + int(split_out[year_indx]))) if current_direction == 'B': qty_list.append(wh_frame['qty'].iloc[i]) elif current_direction == 'S': qty_list.append(-wh_frame['qty'].iloc[i]) else: print(current_direction) elif wh_frame['raw_symbol'].iloc[i] in ['B', 'S']: current_direction = wh_frame['raw_symbol'].iloc[i] else: current_instrument = str(wh_frame['raw_symbol'].iloc[i]) wh_frame = pd.DataFrame() wh_frame['Instrument'] = instrument_list wh_frame['raw_symbol'] = raw_symbol_list wh_frame['strike_price'] = strike_price_list wh_frame['option_type'] = option_type_list wh_frame['qty'] = qty_list wh_frame['ticker_head'] = ticker_head_list wh_frame['ticker'] = ticker_list wh_frame['strike_multiplier'] = [ wh_strike_multiplier.get(x, 1) for x in wh_frame['ticker_head'] ] wh_frame['strike_price'] = round( wh_frame['strike_multiplier'] * wh_frame['strike_price'], 4) wh_frame['instrumet'] = 'F' option_indx = (wh_frame['option_type'] == 'C') | (wh_frame['option_type'] == 'P') wh_frame['instrumet'][option_indx] = 'O' wh_frame['generalized_ticker'] = wh_frame['ticker'] wh_frame['generalized_ticker'][option_indx] = wh_frame['ticker'][option_indx] + '-' + \ wh_frame['option_type'][option_indx] + '-' + \ wh_frame['strike_price'][option_indx].astype(str) wh_frame['generalized_ticker'] = [ x.rstrip('0').rstrip('.') for x in wh_frame['generalized_ticker'] ] return wh_frame[['generalized_ticker', 'qty']]
__author__ = 'kocat_000' import os.path import get_price.get_futures_price as gfp import contract_utilities.contract_meta_info as cmi import my_sql_routines.my_sql_utilities as msu import contract_utilities.expiration as exp from pandas.tseries.offsets import CustomBusinessDay import shared.calendar_utilities as cu import shared.directory_names_aux as dna import pandas as pd import numpy as np import datetime as dt pd.options.mode.chained_assignment = None presaved_futures_data_folder = dna.get_directory_name( ext='presaved_futures_data') dirty_data_points = pd.DataFrame([('BM2006', dt.datetime(2005, 11, 18), True), ('BM2006', dt.datetime(2005, 11, 21), True), ('BM2006', dt.datetime(2005, 11, 25), True), ('BM2006', dt.datetime(2005, 11, 28), True), ('BM2006', dt.datetime(2005, 11, 30), True), ('BM2006', dt.datetime(2005, 12, 1), True), ('BM2006', dt.datetime(2005, 12, 2), True), ('BM2006', dt.datetime(2005, 12, 6), True), ('BM2006', dt.datetime(2005, 12, 8), True), ('BM2006', dt.datetime(2005, 12, 9), True), ('BM2006', dt.datetime(2005, 12, 12), True), ('BM2006', dt.datetime(2005, 12, 13), True), ('BM2006', dt.datetime(2005, 12, 14), True), ('BM2006', dt.datetime(2005, 12, 15), True),
def main(): app = algo.Algo() admin_dir = dna.get_directory_name(ext='admin') risk_file_out = su.read_text_file(file_name=admin_dir + '/RiskParameter.txt') app.bet_size = float(risk_file_out[0]) con = msu.get_my_sql_connection() date_now = cu.get_doubledate() report_date = exp.doubledate_shift_bus_days() report_date_list = [ exp.doubledate_shift_bus_days(shift_in_days=x) for x in range(1, 10) ] overnight_calendars_list = [] for i in range(len(report_date_list)): ocs_output = ocs.generate_overnight_spreads_sheet_4date( date_to=report_date_list[i]) overnight_calendars = ocs_output['overnight_calendars'] overnight_calendars = \ overnight_calendars[overnight_calendars['tickerHead'].isin(['CL', 'HO', 'NG', 'C', 'W', 'KW', 'S', 'SM', 'BO', 'LC', 'LN', 'FC'])] #isin(['CL', 'HO','NG', 'C', 'W', 'KW', 'S', 'SM', 'BO', 'LC', 'LN', 'FC'])] overnight_calendars = overnight_calendars[ (overnight_calendars['ticker1L'] != '') & (overnight_calendars['ticker2L'] != '')] overnight_calendars['back_spread_price'] = np.nan overnight_calendars['front_spread_price'] = np.nan overnight_calendars['mid_ticker_price'] = np.nan overnight_calendars['back_spread_ticker'] = [ overnight_calendars['ticker1'].iloc[x] + '-' + overnight_calendars['ticker2'].iloc[x] for x in range(len(overnight_calendars.index)) ] overnight_calendars['front_spread_ticker'] = [ overnight_calendars['ticker1L'].iloc[x] + '-' + overnight_calendars['ticker2L'].iloc[x] for x in range(len(overnight_calendars.index)) ] overnight_calendars['target_quantity'] = [ min(mth.ceil(app.bet_size / x), app.total_traded_volume_max_before_user_confirmation) for x in overnight_calendars['dollarNoise100'] ] overnight_calendars['alias'] = [ overnight_calendars['ticker1'].iloc[x] + '_' + overnight_calendars['ticker2'].iloc[x] + '_ocs' for x in range(len(overnight_calendars.index)) ] overnight_calendars['total_quantity'] = 0 overnight_calendars['total_risk'] = 0 overnight_calendars['holding_period'] = 0 #overnight_calendars['expiring_position_q'] = 0 overnight_calendars.reset_index(drop=True, inplace=True) overnight_calendars_list.append(overnight_calendars) overnight_calendars = overnight_calendars_list.pop(0) open_strategy_frame = ts.get_filtered_open_strategies( strategy_class_list=['ocs'], as_of_date=date_now) for i in range(len(open_strategy_frame.index)): position_manager_output = pm.get_ocs_position( alias=open_strategy_frame['alias'].iloc[i], as_of_date=date_now, con=con) trades_frame = ts.get_trades_4strategy_alias( alias=open_strategy_frame['alias'].iloc[i], con=con) datetime_now = cu.convert_doubledate_2datetime(date_now) holding_period = (datetime_now - trades_frame['trade_date'].min()).days if (not position_manager_output['empty_position_q']) & ( not position_manager_output['correct_position_q']): print('Check ' + open_strategy_frame['alias'].iloc[i] + ' ! Position may be incorrect') elif position_manager_output['correct_position_q']: ticker_head = cmi.get_contract_specs( position_manager_output['sorted_position'] ['ticker'].iloc[0])['ticker_head'] position_name = '' if position_manager_output['scale'] > 0: position_name = ticker_head + '_long' else: position_name = ticker_head + '_short' app.ocs_portfolio.order_send(ticker=position_name, qty=abs( position_manager_output['scale'])) app.ocs_portfolio.order_fill(ticker=position_name, qty=abs( position_manager_output['scale'])) ticker1 = position_manager_output['sorted_position'][ 'ticker'].iloc[0] ticker2 = position_manager_output['sorted_position'][ 'ticker'].iloc[1] selection_indx = overnight_calendars[ 'back_spread_ticker'] == ticker1 + '-' + ticker2 if sum(selection_indx) == 1: overnight_calendars.loc[ selection_indx, 'total_quantity'] = position_manager_output['scale'] overnight_calendars.loc[ selection_indx, 'total_risk'] = position_manager_output[ 'scale'] * overnight_calendars.loc[selection_indx, 'dollarNoise100'] overnight_calendars.loc[ selection_indx, 'alias'] = open_strategy_frame['alias'].iloc[i] overnight_calendars.loc[selection_indx, 'holding_period'] = holding_period app.ocs_risk_portfolio.order_send( ticker=position_name, qty=abs( position_manager_output['scale'] * overnight_calendars.loc[selection_indx, 'dollarNoise100'].values[0])) app.ocs_risk_portfolio.order_fill( ticker=position_name, qty=abs( position_manager_output['scale'] * overnight_calendars.loc[selection_indx, 'dollarNoise100'].values[0])) else: for j in range(len(overnight_calendars_list)): overnight_calendars_past = overnight_calendars_list[j] selection_indx = overnight_calendars_past[ 'back_spread_ticker'] == ticker1 + '-' + ticker2 if sum(selection_indx) == 1: overnight_calendars_past.loc[ selection_indx, 'total_quantity'] = position_manager_output[ 'scale'] overnight_calendars_past.loc[ selection_indx, 'total_risk'] = position_manager_output[ 'scale'] * overnight_calendars_past.loc[ selection_indx, 'dollarNoise100'] overnight_calendars_past.loc[ selection_indx, 'alias'] = open_strategy_frame['alias'].iloc[i] overnight_calendars_past.loc[ selection_indx, 'holding_period'] = holding_period app.ocs_risk_portfolio.order_send( ticker=position_name, qty=abs(position_manager_output['scale'] * overnight_calendars_past.loc[ selection_indx, 'dollarNoise100'].values[0])) app.ocs_risk_portfolio.order_fill( ticker=position_name, qty=abs(position_manager_output['scale'] * overnight_calendars_past.loc[ selection_indx, 'dollarNoise100'].values[0])) if j > 1: overnight_calendars_past.loc[ selection_indx, 'butterflyMean'] = np.nan overnight_calendars_past.loc[ selection_indx, 'butterflyNoise'] = np.nan overnight_calendars = overnight_calendars.append( overnight_calendars_past[selection_indx]) break overnight_calendars.reset_index(drop=True, inplace=True) overnight_calendars['working_order_id'] = np.nan spread_ticker_list = list( set(overnight_calendars['back_spread_ticker']).union( overnight_calendars['front_spread_ticker'])) back_spread_ticker_list = list(overnight_calendars['back_spread_ticker']) theme_name_list = set([ x + '_long' for x in back_spread_ticker_list ]).union(set([x + '_short' for x in back_spread_ticker_list])) ocs_alias_portfolio = aup.portfolio(ticker_list=theme_name_list) for i in range(len(overnight_calendars.index)): if overnight_calendars.loc[i, 'total_quantity'] > 0: position_name = overnight_calendars.loc[ i, 'back_spread_ticker'] + '_long' ocs_alias_portfolio.order_send( ticker=position_name, qty=overnight_calendars.loc[i, 'total_quantity']) ocs_alias_portfolio.order_fill( ticker=position_name, qty=overnight_calendars.loc[i, 'total_quantity']) elif overnight_calendars.loc[i, 'total_quantity'] < 0: position_name = overnight_calendars.loc[ i, 'back_spread_ticker'] + '_short' ocs_alias_portfolio.order_send( ticker=position_name, qty=-overnight_calendars.loc[i, 'total_quantity']) ocs_alias_portfolio.order_fill( ticker=position_name, qty=-overnight_calendars.loc[i, 'total_quantity']) app.price_request_dictionary['spread'] = spread_ticker_list app.price_request_dictionary['outright'] = overnight_calendars[ 'ticker1'].values app.overnight_calendars = overnight_calendars app.open_strategy_list = list(open_strategy_frame['alias']) app.ocs_alias_portfolio = ocs_alias_portfolio app.ticker_list = list( set(overnight_calendars['ticker1']).union( overnight_calendars['ticker2']).union( set(overnight_calendars['ticker1L'])).union( set(overnight_calendars['ticker2L']))) app.output_dir = ts.create_strategy_output_dir(strategy_class='ocs', report_date=report_date) app.log = lg.get_logger(file_identifier='ib_ocs', log_level='INFO') app.con = con app.pnl_frame = tpm.get_daily_pnl_snapshot(as_of_date=report_date, name='final') print('Emre') app.connect(client_id=2) app.run()