def __init__(self, host='localhost', port=4001, client_id=101, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(hours=1)): self.moving_window_period = moving_window_period self.chart = Chart() self.ib_util = IBUtil() # Store parameters for this model self.strategy_params = StrategyParameters(evaluation_time_secs, resample_interval_secs) self.stocks_data = {} # Dictionary storing StockData objects. self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() # Use ibConnection() for TWS, or create connection for API Gateway self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler)
def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(seconds=60)): self.moving_window_period = moving_window_period # self.chart = Chart() self.ib_util = IBUtil() # Store parameters for this model # self.strategy_params = StrategyParameters(evaluation_time_secs, # resample_interval_secs) self.stocks_data = {} # Dictionary storing StockData objects. self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() #addition for hdf store self.data_path = os.path.normpath("/Users/maxime_back/Documents/avocado/data.csv") self.ohlc_path = os.path.normpath("/Users/maxime_back/Documents/avocado/ohlc.csv") self.last_trim = dt.datetime.now() # Use ibConnection() for TWS, or create connection for API Gateway self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler)
def __init__(self, host='localhost', port=4001, client_id=101, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(hours=1)): self.moving_window_period = moving_window_period self.chart = Chart() self.ib_util = IBUtil() self.strategy_params = StrategyParameters(evaluation_time_secs, resample_interval_secs) self.stocks_data = {} self.symbols = None self.account_code = "" self.prices = None self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler)
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=101, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(hours=1)): self.moving_window_period = moving_window_period self.chart = Chart() self.ib_util = IBUtil() # Store parameters for this model self.strategy_params = StrategyParameters(evaluation_time_secs, resample_interval_secs) self.stocks_data = {} # Dictionary storing StockData objects. self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() # Use ibConnection() for TWS, or create connection for API Gateway self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler) def __perform_trade_logic(self): """ This part is the 'secret-sauce' where actual trades takes place. My take is that great experience, good portfolio construction, and together with robust backtesting will make your strategy viable. GOOD PORTFOLIO CONTRUCTION CAN SAVE YOU FROM BAD RESEARCH, BUT BAD PORTFOLIO CONSTRUCTION CANNOT SAVE YOU FROM GREAT RESEARCH This trade logic uses volatility ratio and beta as our indicators. - volatility ratio > 1 :: uptrend, volatility ratio < 1 :: downtrend - beta is calculated as: mean(price A) / mean(price B) We use the assumption that prive levels will mean-revert. Expected price A = beta x price B Consider other methods of identifying our trade logic: - current trend - current regime - detect structural breaks """ volatility_ratio = self.strategy_params.get_volatility_ratio() is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1 is_overbought, is_oversold = self.__is_overbought_or_oversold() # Our final trade signals is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \ (is_down_trend and is_overbought) # Use account position details symbol_a = self.symbols[0] position = self.stocks_data[symbol_a].position is_position_closed, is_short, is_long = \ (position == 0), (position < 0), (position > 0) upnl, rpnl = self.__calculate_pnls() # Display to terminal dynamically signal_text = \ "BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE" console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \ (dt.datetime.now(), signal_text, position, upnl, rpnl) sys.stdout.write(console_output) sys.stdout.flush() if is_position_closed and is_sell_signal: print "==================================" print "OPEN SHORT POSIITON: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) elif is_position_closed and is_buy_signal: print "==================================" print "OPEN LONG POSIITON: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_short and is_buy_signal: print "==================================" print "CLOSE SHORT POSITION: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_long and is_sell_signal: print "==================================" print "CLOSE LONG POSITION: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) def __recalculate_strategy_parameters_at_interval(self): """ Consider re-evaluation of parameters on: - regime shifts - structural breaks """ if self.strategy_params.is_evaluation_time_elapsed(): self.__calculate_strategy_params() self.strategy_params.set_new_evaluation_time() print '[%s] === Beta re-evaluated ===' % dt.datetime.now() def __calculate_strategy_params(self): """ Here, we are calculating beta and volatility ratio for our signal indicators. Consider calculating other statistics here: - stddevs of errs - correlations - co-integration """ [symbol_a, symbol_b] = self.symbols filled_prices = self.prices.fillna(method='ffill') resampled = filled_prices.resample( self.strategy_params.resample_interval_secs, fill_method='ffill')\ .dropna() mean = resampled.mean() beta = mean[symbol_a] / mean[symbol_b] stddevs = resampled.pct_change().dropna().std() volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b] self.strategy_params.add_indicators(beta, volatility_ratio) def __register_data_handlers(self, tick_event_handler, universal_event_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) def __init_stocks_data(self, symbols): self.symbols = symbols self.prices = pd.DataFrame(columns=symbols) # Init price storage for stock_symbol in symbols: contract = self.ib_util.create_stock_contract(stock_symbol) self.stocks_data[stock_symbol] = StockData(contract) def __request_streaming_data(self, ib_conn): # Stream market data for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): ib_conn.reqMktData(index, stock_data.contract, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) time.sleep(1) # Stream account updates ib_conn.reqAccountUpdates(True, self.account_code) def __request_historical_data(self, ib_conn): self.lock.acquire() try: for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): stock_data.is_storing_data = True ib_conn.reqHistoricalData( index, stock_data.contract, time.strftime(datatype.DATE_TIME_FORMAT), datatype.DURATION_1_HR, datatype.BAR_SIZE_5_SEC, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) finally: self.lock.release() def __wait_for_download_completion(self): is_waiting = True while is_waiting: is_waiting = False self.lock.acquire() try: for symbol in self.stocks_data.keys(): if self.stocks_data[symbol].is_storing_data: is_waiting = True finally: self.lock.release() if is_waiting: time.sleep(1) def __place_spread_order(self, qty): [symbol_a, symbol_b] = self.symbols self.__send_order(symbol_a, qty) self.__send_order(symbol_b, -qty) def __send_order(self, symbol, qty): stock_data = self.stocks_data[symbol] order = self.ib_util.create_stock_order(abs(qty), qty > 0) self.conn.placeOrder(self.__generate_order_id(), stock_data.contract, order) stock_data.add_to_position(qty) def __generate_order_id(self): next_order_id = self.order_id self.order_id += 1 return next_order_id def __is_overbought_or_oversold(self): [symbol_a, symbol_b] = self.symbols leg_a_last_price = self.prices[symbol_a].values[-1] leg_b_last_price = self.prices[symbol_b].values[-1] expected_leg_a_price = \ leg_b_last_price * self.strategy_params.get_beta() is_overbought = \ leg_a_last_price < expected_leg_a_price # Cheaper than expected is_oversold = \ leg_a_last_price > expected_leg_a_price # Higher than expected return is_overbought, is_oversold def __on_portfolio_update(self, msg): for key, stock_data in self.stocks_data.iteritems(): if stock_data.contract.m_symbol == msg.contract.m_symbol: stock_data.update_position(msg.position, msg.marketPrice, msg.marketValue, msg.averageCost, msg.unrealizedPNL, msg.realizedPNL, msg.accountName) return def __calculate_pnls(self): upnl, rpnl = 0, 0 for key, stock_data in self.stocks_data.iteritems(): upnl += stock_data.unrealized_pnl rpnl += stock_data.realized_pnl return upnl, rpnl def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: self.account_code = msg.accountsList elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID: self.order_id = msg.orderId else: print msg def __on_historical_data(self, msg): print msg ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed(ticker_index) else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self, ticker_index): self.lock.acquire() try: symbol = self.symbols[ticker_index] self.stocks_data[symbol].is_storing_data = False finally: self.lock.release() def __add_historical_data(self, ticker_index, msg): timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT) self.__add_market_data(ticker_index, timestamp, msg.close) def __on_tick_event(self, msg): ticker_id = msg.tickerId field_type = msg.field # Store information from last traded price if field_type == datatype.FIELD_LAST_PRICE: last_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), last_price) self.__trim_data_series() # Post-bootstrap - make trading decisions if self.strategy_params.is_bootstrap_completed(): self.__recalculate_strategy_parameters_at_interval() self.__perform_trade_logic() self.__update_charts() def __add_market_data(self, ticker_index, timestamp, price): symbol = self.symbols[ticker_index] self.prices.loc[timestamp, symbol] = float(price) self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) def __update_charts(self): if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0: self.chart.display_chart(self.prices, self.strategy_params.indicators) def __trim_data_series(self): cutoff_timestamp = dt.datetime.now() - self.moving_window_period self.prices = self.prices[self.prices.index >= cutoff_timestamp] self.strategy_params.trim_indicators_series(cutoff_timestamp) @staticmethod def __print_elapsed_time(start_time): elapsed_time = time.time() - start_time print "Completed in %.3f seconds." % elapsed_time def __cancel_market_data_request(self): for i, symbol in enumerate(self.symbols): self.conn.cancelMktData(i) time.sleep(1) def start(self, symbols, trade_qty): print "HFT model started." self.trade_qty = trade_qty self.conn.connect() # Get IB connection object self.__init_stocks_data(symbols) self.__request_streaming_data(self.conn) print "Bootstrapping the model..." start_time = time.time() self.__request_historical_data(self.conn) self.__wait_for_download_completion() self.strategy_params.set_bootstrap_completed() self.__print_elapsed_time(start_time) print "Calculating strategy parameters..." start_time = time.time() self.__calculate_strategy_params() self.__print_elapsed_time(start_time) print "Trading started." try: self.__update_charts() while True: time.sleep(1) except Exception, e: print "Exception:", e print "Cancelling...", self.__cancel_market_data_request() print "Disconnecting..." self.conn.disconnect() time.sleep(1) print "Disconnected."
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=101, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(hours=1)): self.moving_window_period = moving_window_period self.chart = Chart() self.ib_util = IBUtil() # Store parameters for this model self.strategy_params = StrategyParameters(evaluation_time_secs, resample_interval_secs) self.stocks_data = {} # Dictionary storing StockData objects. self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() # Use ibConnection() for TWS, or create connection for API Gateway self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler) def __perform_trade_logic(self): """ This part is the 'secret-sauce' where actual trades takes place. My take is that great experience, good portfolio construction, and together with robust backtesting will make your strategy viable. GOOD PORTFOLIO CONTRUCTION CAN SAVE YOU FROM BAD RESEARCH, BUT BAD PORTFOLIO CONSTRUCTION CANNOT SAVE YOU FROM GREAT RESEARCH This trade logic uses volatility ratio and beta as our indicators. - volatility ratio > 1 :: uptrend, volatility ratio < 1 :: downtrend - beta is calculated as: mean(price A) / mean(price B) We use the assumption that prive levels will mean-revert. Expected price A = beta x price B Consider other methods of identifying our trade logic: - current trend - current regime - detect structural breaks """ volatility_ratio = self.strategy_params.get_volatility_ratio() is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1 is_overbought, is_oversold = self.__is_overbought_or_oversold() # Our final trade signals is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \ (is_down_trend and is_overbought) # Use account position details symbol_a = self.symbols[0] position = self.stocks_data[symbol_a].position is_position_closed, is_short, is_long = \ (position == 0), (position < 0), (position > 0) upnl, rpnl = self.__calculate_pnls() # Display to terminal dynamically signal_text = \ "BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE" console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \ (dt.datetime.now(), signal_text, position, upnl, rpnl) sys.stdout.write(console_output) sys.stdout.flush() if is_position_closed and is_sell_signal: print "==================================" print "OPEN SHORT POSIITON: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) elif is_position_closed and is_buy_signal: print "==================================" print "OPEN LONG POSIITON: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_short and is_buy_signal: print "==================================" print "CLOSE SHORT POSITION: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_long and is_sell_signal: print "==================================" print "CLOSE LONG POSITION: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) def __recalculate_strategy_parameters_at_interval(self): """ Consider re-evaluation of parameters on: - regime shifts - structural breaks """ if self.strategy_params.is_evaluation_time_elapsed(): self.__calculate_strategy_params() self.strategy_params.set_new_evaluation_time() print '[%s] === Beta re-evaluated ===' % dt.datetime.now() def __calculate_strategy_params(self): """ Here, we are calculating beta and volatility ratio for our signal indicators. Consider calculating other statistics here: - stddevs of errs - correlations - co-integration """ [symbol_a, symbol_b] = self.symbols filled_prices = self.prices.fillna(method='ffill') resampled = filled_prices.resample( self.strategy_params.resample_interval_secs, fill_method='ffill')\ .dropna() mean = resampled.mean() beta = mean[symbol_a] / mean[symbol_b] stddevs = resampled.pct_change().dropna().std() volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b] self.strategy_params.add_indicators(beta, volatility_ratio) def __register_data_handlers(self, tick_event_handler, universal_event_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) def __init_stocks_data(self, symbols): self.symbols = symbols self.prices = pd.DataFrame(columns=symbols) # Init price storage for stock_symbol in symbols: contract = self.ib_util.create_stock_contract(stock_symbol) self.stocks_data[stock_symbol] = StockData(contract) def __request_streaming_data(self, ib_conn): # Stream market data for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): ib_conn.reqMktData(index, stock_data.contract, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) time.sleep(1) # Stream account updates ib_conn.reqAccountUpdates(True, self.account_code) def __request_historical_data(self, ib_conn): self.lock.acquire() try: for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): stock_data.is_storing_data = True ib_conn.reqHistoricalData( index, stock_data.contract, time.strftime(datatype.DATE_TIME_FORMAT), datatype.DURATION_1_HR, datatype.BAR_SIZE_5_SEC, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) finally: self.lock.release() def __wait_for_download_completion(self): is_waiting = True while is_waiting: is_waiting = False self.lock.acquire() try: for symbol in self.stocks_data.keys(): if self.stocks_data[symbol].is_storing_data: is_waiting = True finally: self.lock.release() if is_waiting: time.sleep(1) def __place_spread_order(self, qty): [symbol_a, symbol_b, symbol_c] = self.symbols self.__send_order(symbol_a, qty) time.sleep(3) self.__send_order(symbol_b, qty) time.sleep(3) self.__send_order(symbol_c, -qty) time.sleep(3) def __send_order(self, symbol, qty): stock_data = self.stocks_data[symbol] order = self.ib_util.create_stock_order(abs(qty), qty > 0) self.conn.placeOrder(self.__generate_order_id(), stock_data.contract, order) stock_data.add_to_position(qty) def __generate_order_id(self): next_order_id = self.order_id self.order_id += 1 return next_order_id def __is_overbought_or_oversold(self): [symbol_a, symbol_b] = self.symbols leg_a_last_price = self.prices[symbol_a].values[-1] leg_b_last_price = self.prices[symbol_b].values[-1] expected_leg_a_price = \ leg_b_last_price * self.strategy_params.get_beta() is_overbought = \ leg_a_last_price < expected_leg_a_price # Cheaper than expected is_oversold = \ leg_a_last_price > expected_leg_a_price # Higher than expected return is_overbought, is_oversold def __on_portfolio_update(self, msg): for key, stock_data in self.stocks_data.iteritems(): if stock_data.contract.m_symbol == msg.contract.m_symbol: stock_data.update_position(msg.position, msg.marketPrice, msg.marketValue, msg.averageCost, msg.unrealizedPNL, msg.realizedPNL, msg.accountName) return def __calculate_pnls(self): upnl, rpnl = 0, 0 for key, stock_data in self.stocks_data.iteritems(): upnl += stock_data.unrealized_pnl rpnl += stock_data.realized_pnl return upnl, rpnl def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: self.account_code = msg.accountsList elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID: self.order_id = msg.orderId else: print msg def __on_historical_data(self, msg): print msg ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed(ticker_index) else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self, ticker_index): self.lock.acquire() try: symbol = self.symbols[ticker_index] self.stocks_data[symbol].is_storing_data = False finally: self.lock.release() def __add_historical_data(self, ticker_index, msg): timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT) self.__add_market_data(ticker_index, timestamp, msg.close) def __on_tick_event(self, msg): ticker_id = msg.tickerId field_type = msg.field # Store information from last traded price if field_type == datatype.FIELD_LAST_PRICE: last_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), last_price) self.__trim_data_series() # Post-bootstrap - make trading decisions if self.strategy_params.is_bootstrap_completed(): self.__recalculate_strategy_parameters_at_interval() self.__perform_trade_logic() self.__update_charts() def __add_market_data(self, ticker_index, timestamp, price): symbol = self.symbols[ticker_index] self.prices.loc[timestamp, symbol] = float(price) self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) def __update_charts(self): if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0: self.chart.display_chart(self.prices, self.strategy_params.indicators) def __trim_data_series(self): cutoff_timestamp = dt.datetime.now() - self.moving_window_period self.prices = self.prices[self.prices.index >= cutoff_timestamp] self.strategy_params.trim_indicators_series(cutoff_timestamp) @staticmethod def __print_elapsed_time(start_time): elapsed_time = time.time() - start_time print "Completed in %.3f seconds." % elapsed_time def __cancel_market_data_request(self): for i, symbol in enumerate(self.symbols): self.conn.cancelMktData(i) time.sleep(1) def start(self, symbols, trade_qty): print "HFT model started." self.trade_qty = trade_qty self.conn.connect() # Get IB connection object self.__init_stocks_data(symbols) self.__request_streaming_data(self.conn) print "Bootstrapping the model..." start_time = time.time() self.__request_historical_data(self.conn) self.__wait_for_download_completion() self.strategy_params.set_bootstrap_completed() self.__print_elapsed_time(start_time) print "Calculating strategy parameters..." start_time = time.time() self.__calculate_strategy_params() self.__print_elapsed_time(start_time) print "Trading started." try: self.__update_charts() while True: time.sleep(1) except Exception, e: print "Exception:", e print "Cancelling...", self.__cancel_market_data_request() print "Disconnecting..." self.conn.disconnect() time.sleep(1) print "Disconnected."
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(seconds=60)): self.moving_window_period = moving_window_period # self.chart = Chart() self.ib_util = IBUtil() # Store parameters for this model # self.strategy_params = StrategyParameters(evaluation_time_secs, # resample_interval_secs) self.stocks_data = {} # Dictionary storing StockData objects. self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() #addition for hdf store self.data_path = os.path.normpath("/Users/maxime_back/Documents/avocado/data.csv") self.ohlc_path = os.path.normpath("/Users/maxime_back/Documents/avocado/ohlc.csv") self.last_trim = dt.datetime.now() # Use ibConnection() for TWS, or create connection for API Gateway self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler) def __register_data_handlers(self, tick_event_handler, universal_event_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) def __init_stocks_data(self, symbols): self.symbols = symbols #here we'll store tick and size instead of multiple "symbols" self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size")) # Init price storage if not os.path.exists(self.data_path): self.prices.to_csv(self.data_path) self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage if not os.path.exists(self.ohlc_path): self.ohlc.to_csv(self.ohlc_path) print "checked for csv file" #Now I have only one "symbol" stock_symbol = self.symbols contract = self.ib_util.create_stock_contract(stock_symbol) self.stocks_data[stock_symbol] = StockData(contract) print "contracts initated" def __request_streaming_data(self, ib_conn): # Stream market data. Of note: this enumerate can probably be simplified #cannt be bothered for now for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): ib_conn.reqMktData(index, stock_data.contract, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) time.sleep(5) # Stream account updates DEACTIVATED FOR NOW # ib_conn.reqAccountUpdates(True, self.account_code) def __request_historical_data(self, ib_conn): self.lock.acquire() try: for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): stock_data.is_storing_data = True ib_conn.reqHistoricalData( index, stock_data.contract, time.strftime(datatype.DATE_TIME_FORMAT), datatype.DURATION_1_DAY, datatype.BAR_SIZE_1_MIN, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) finally: self.lock.release() self.last_trim = dt.datetime.now() # def __wait_for_download_completion(self): # is_waiting = True # while is_waiting: # is_waiting = False # # self.lock.acquire() # try: # for symbol in self.stocks_data.keys(): # if self.stocks_data[symbol].is_storing_data: # is_waiting = True # finally: # self.lock.release() # # if is_waiting: # time.sleep(1) # def __on_portfolio_update(self, msg): # for key, stock_data in self.stocks_data.iteritems(): # if stock_data.contract.m_symbol == msg.contract.m_symbol: # stock_data.update_position(msg.position, # msg.marketPrice, # msg.marketValue, # msg.averageCost, # msg.unrealizedPNL, # msg.realizedPNL, # msg.accountName) # return # # def __calculate_pnls(self): # upnl, rpnl = 0, 0 # for key, stock_data in self.stocks_data.iteritems(): # upnl += stock_data.unrealized_pnl # rpnl += stock_data.realized_pnl # return upnl, rpnl def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: pass # self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: pass # self.account_code = msg.accountsList elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID: self.order_id = msg.orderId else: print msg def __on_historical_data(self, msg): print msg ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed(ticker_index) else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self, ticker_index): self.lock.acquire() try: symbol = self.symbols#[ticker_index] self.stocks_data[symbol].is_storing_data = False finally: self.lock.release() def __add_historical_data(self, ticker_index, msg): timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT) self.__add_ohlc_data(ticker_index, timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count) def __add_ohlc_data(self, ticker_index, timestamp, op, hi ,lo,close,vol,cnt ): self.ohlc.loc[timestamp, "open"] = float(op) self.ohlc.loc[timestamp, "high"] = float(hi) self.ohlc.loc[timestamp, "low"] = float(lo) self.ohlc.loc[timestamp, "close"] = float(close) self.ohlc.loc[timestamp, "volume"] = float(vol) self.ohlc.loc[timestamp, "count"] = float(cnt) def __on_tick_event(self, msg): ticker_id = msg.tickerId field_type = msg.field # print field_type # Store information from last traded price if field_type == datatype.FIELD_LAST_PRICE: last_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), last_price, 1) if field_type == datatype.FIELD_LAST_SIZE: last_size = msg.size self.__add_market_data(ticker_id, dt.datetime.now(), last_size, 2) if field_type == datatype.FIELD_ASK_PRICE: ask_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), ask_price, 3) if field_type == datatype.FIELD_ASK_SIZE: ask_size = msg.size self.__add_market_data(ticker_id, dt.datetime.now(), ask_size, 4) if field_type == datatype.FIELD_BID_PRICE: bid_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), bid_price, 5) if field_type == datatype.FIELD_BID_SIZE: bid_size = msg.size self.__add_market_data(ticker_id, dt.datetime.now(), bid_size, 6) #now to trim the serie every 60 second if dt.datetime.now() > self.last_trim + self.moving_window_period: self.__trim_data_series() # Post-bootstrap - make trading decisions # if self.strategy_params.is_bootstrap_completed(): # self.__recalculate_strategy_parameters_at_interval() # self.__perform_trade_logic() # self.__update_charts() def __add_market_data(self, ticker_index, timestamp, value, col): if col == 1: self.prices.loc[timestamp, "price"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values ## Not sure how to handle this one self.prices.sort_index(inplace=True) elif col ==2: self.prices.loc[timestamp, "size"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) elif col ==3: self.prices.loc[timestamp, "ask_price"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) elif col ==4: self.prices.loc[timestamp, "ask_size"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) elif col ==5: self.prices.loc[timestamp, "bid_price"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) elif col ==6: self.prices.loc[timestamp, "bid_size"] = float(value) # self.prices = self.prices.fillna(method='ffill') # Clear NaN values self.prices.sort_index(inplace=True) def __stream_to_ohlc(self): # stream = stream[["price","size"]] # stream.is_copy = False # stream.dropna(inplace=True, how='all') try: new_ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # very likely fuckery to be checked at the cutoff t_stmp = self.prices.first_valid_index().replace(second=0, microsecond=0) new_ohlc.loc[t_stmp, "open"] = float(self.prices['price'].dropna().head(1)) new_ohlc.loc[t_stmp, "close"] = float(self.prices['price'].dropna().tail(1)) new_ohlc.loc[t_stmp, "high"] = float(self.prices['price'].max()) new_ohlc.loc[t_stmp, "low"] = float(self.prices['price'].min()) new_ohlc.loc[t_stmp, "low"] = float(self.prices['price'].min()) new_ohlc.loc[t_stmp, "volume"] = float(self.prices['size'].sum()) new_ohlc.loc[t_stmp, "count"] = float(self.prices['size'].count()) return new_ohlc except Exception, e: print "f**k:", e print self.prices
def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): logging.basicConfig(format='%(asctime)s %(message)s') self.test_logger = logging.getLogger('hftModelLogger') self.test_logger.setLevel(logging.INFO) self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6) self.timekeeper = None self.parser = None self.execution = None self.strategy = None self.handler = None self.data_path = os.path.normpath( os.path.join(os.path.curdir, "data.csv")) self.ohlc_path = os.path.normpath( os.path.join(os.path.curdir, "ohlc.csv")) self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None self.cur_mean = None self.cur_sd = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) if not self.test: self.handler = ExecutionHandler(self.conn) #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) else: self.order_template = self.handler.create_contract( settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) self.signal = None self.state = None
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): logging.basicConfig(format='%(asctime)s %(message)s') self.test_logger = logging.getLogger('hftModelLogger') self.test_logger.setLevel(logging.INFO) self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6) self.timekeeper = None self.parser = None self.execution = None self.strategy = None self.handler = None self.data_path = os.path.normpath( os.path.join(os.path.curdir, "data.csv")) self.ohlc_path = os.path.normpath( os.path.join(os.path.curdir, "ohlc.csv")) self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None self.cur_mean = None self.cur_sd = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) if not self.test: self.handler = ExecutionHandler(self.conn) #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) else: self.order_template = self.handler.create_contract( settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) self.signal = None self.state = None def _market_is_open(self): tz_cme = pytz.timezone('America/Chicago') cme_now = self.now.astimezone(tz_cme) if cme_now.weekday() in [ 0, 1, 2, 3 ] and (cme_now >= cme_now.replace(hour=17, minute=0, second=0) or cme_now < cme_now.replace(hour=16, minute=0, second=0)): self.event_market_on.set() elif cme_now.weekday() is 4 and cme_now < cme_now.replace( hour=16, minute=0, second=0): self.event_market_on.set() elif cme_now.weekday() is 6 and cme_now >= cme_now.replace( hour=17, minute=0, second=0): self.event_market_on.set() else: self.event_market_on.clear() # print "market is on:" # print self.event_market_on.is_set() def time_keeper(self): while True: print "executing and trading flag set?" print self.handler.executing.is_set() print self.handler.trading.is_set() self.now = pytz.timezone('Singapore').localize(dt.datetime.now()) self._market_is_open() # OHLC call if self.last_trim is None: self.last_trim = self.now if self.now > self.last_trim + dt.timedelta(minutes=1): if self.parser.running(): self.test_logger.error( "parser thread is alive - timekeeper") if self.strategy.running(): self.test_logger.error( "strategy thread is alive - timekeeper") if self.execution.running(): self.test_logger.error( "execution thread is alive - timekeeper") else: self.test_logger.error( "!!!execution thread is DEAD- timekeeper") self.test_logger.error("timekeeper minute call") try: self.__request_historical_data(self.conn, initial=False) except: self.test_logger.error( "req of update historicals failed - timekeep/hft") try: self.__run_indicators(self.ohlc) except: self.test_logger.error( "runnning indicators failed - timekeep/hft") self.conn.reqPositions() self.update_norm_params() if self.test: print self.ohlc.tail() time.sleep( 5 ) #TODO horrible, horrible, but can't be bothered with a lock right now # ML Call if self.last_ml_call is None: self.last_ml_call = self.now if self.now > self.last_ml_call + dt.timedelta(minutes=5): #logging.DEBUG("ML Call") #logging.DEBUG(str(self.ohlc)) try: self.flag = self.ml.call_ml(self.ohlc) self.handler.flag = self.flag # this is stupid todo why why why self.test_logger.error("I believe we will " + self.handler.flag) self.last_ml_call = self.now except: self.test_logger.error( "!!! Call ML failed - timekeeper/hft") time.sleep(1) def __register_data_handlers(self, tick_event_handler, universal_event_handler, order_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) self.conn.register(order_handler, ib_message_type.position, ib_message_type.nextValidId, ib_message_type.orderStatus, ib_message_type.openOrder, ib_message_type.error) def __init_stocks_data(self, symbols): #todo brutal hack-through self.symbols = symbols #here we'll store tick and size instead of multiple "symbols" self.prices = pd.DataFrame(columns=( "price", "size", "ask_price", "ask_size", "bid_price", "bid_size")) #todomoved to execution # Init price storage if not os.path.exists(self.data_path): self.prices.to_csv(self.data_path) self.ohlc = pd.DataFrame(columns=("open", "high", "low", "close", "volume", "count")) # Init ohlc storage if not os.path.exists(self.ohlc_path): self.ohlc.to_csv(self.ohlc_path) print "checked for csv file" #Now I have only one "symbol" TODO: clean that stuff stock_symbol = self.symbols contract = self.ib_util.create_stock_contract(stock_symbol) #self.stocks_data[stock_symbol] = StockData(contract) #this is redundant but required to scaffold/test def create_contract(self, symbol, sec_type, exch, expiry, curr): """Create a Contract object defining what will be purchased, at which exchange and in which currency. symbol - The ticker symbol for the contract sec_type - The security type for the contract ('FUT' = Future) exch - The exchange to carry out the contract on prim_exch - The primary exchange to carry out the contract on curr - The currency in which to purchase the contract""" contract = Contract() contract.m_symbol = symbol contract.m_secType = sec_type contract.m_exchange = exch contract.m_expiry = expiry contract.m_currency = curr return contract def __request_streaming_data(self, ib_conn): # Stream market data. ib_conn.reqMktData(1, self.order_template, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) # time.sleep(5) def __request_historical_data(self, ib_conn, initial=True): """ the same method can be used for scheduled calls""" # self.ohlc_ok.acquire() if initial: duration = datatype.DURATION_2_HR else: duration = datatype.DURATION_1_MIN ib_conn.reqHistoricalData(1, self.order_template, time.strftime(datatype.DATE_TIME_FORMAT), duration, datatype.BAR_SIZE_1_MIN, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: pass #self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: pass else: print msg def null_handler(self, msg): pass def __on_historical_data(self, msg): ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed() else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self): #self.lock.release() self.last_trim = self.ohlc.index[-1] + self.moving_window_period print "trim time properly set now %s" % self.last_trim print "start position is:" + str(self.handler.position) self.__run_indicators(self.ohlc) self.ohlc.to_csv(self.ohlc_path) def __add_historical_data(self, ticker_index, msg): if self.test: print "adding histo line" timestamp = pytz.timezone('Singapore').localize( dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)) self.__add_ohlc_data(timestamp, msg.open, msg.high, msg.low, msg.close, msg.volume, msg.count) def __add_ohlc_data(self, timestamp, op, hi, lo, close, vol, cnt): self.ohlc.loc[timestamp, "open"] = float(op) self.ohlc.loc[timestamp, "high"] = float(hi) self.ohlc.loc[timestamp, "low"] = float(lo) self.ohlc.loc[timestamp, "close"] = float(close) self.ohlc.loc[timestamp, "volume"] = float(vol) self.ohlc.loc[timestamp, "count"] = float(cnt) # def __run_indicators(self, ohlc): #hardcoding ML munging parameters now ohlc['returns'] = ta.ROC(np.asarray(ohlc['close']).astype(float)) ohlc['sma'] = ta.SMA(np.asarray(ohlc['close']).astype(float), 10) ohlc['lma'] = ta.SMA(np.asarray(ohlc['close']).astype(float), 120) ohlc['rsi'] = ta.RSI(np.asarray(ohlc['close']).astype(float)) ohlc['atr'] = ta.ATR( np.asarray(ohlc['high']).astype(float), np.asarray(ohlc['low']).astype(float), np.asarray(ohlc['close']).astype(float), 10) ohlc['monday'] = np.where(ohlc.index.weekday == 0, 1, 0) #bellow is because I don't know how to np.where with multiple conditions ohlc['roll'] = np.where(ohlc.index.month % 3 == 0, 1, 0) #hackish as balls: # ohlc.index = ohlc.index.tz_localize("Singapore") ohlc["busy"] = np.where( ohlc.index.tz_convert("America/Chicago").hour >= 9, np.where( ohlc.index.tz_convert("America/Chicago").hour <= 14, 1, 0), 0) def update_norm_params(self): try: prices = self.handler.prices["price"] except: self.test_logger.error( "getting price from handler failed - update norm/hft") sgp_tz = pytz.timezone('Singapore') try: prices.index = prices.index.tz_localize(sgp_tz) #prices.to_pick (os.path.join(os.path.curdir,"prices_f_norm.csv")) except: self.test_logger.error( "prices localization failed - update norm/hft") # try: prices = prices.dropna() except: self.test_logger.error("dropna failed - update norm/hft") try: if prices.index.max() - prices.index.min() > dt.timedelta( seconds=60) and self.last_trim is not None: prices = prices[ prices.index > prices.index.max() - dt.timedelta( seconds=60)] #todo no trim of prices as of now print "trimmed prices" print prices #prices.to_pickle(os.path.join(os.path.curdir(), "prices.pkl")) except: self.test_logger.error( "minute trim of prices failed - update norm/hft") if len(prices) != 0: last_price = prices.iloc[-1] try: my_cur_mean = round(np.mean(prices), 2) self.handler.cur_mean = my_cur_mean except: self.test_logger.error("mean update failed- update norm/hft") try: prices = prices.diff() prices = prices.dropna() prices = prices**2 except: self.test_logger.error("diff/square failed- update norm/hft") tdiffs = list() try: for i in range(1, len(prices)): tdiffs.append((prices.index[i] - prices.index[i - 1]).total_seconds()) prices = prices.ix[1:] print prices except: self.test_logger.error( "time diffs creation failed - update norm/hft") try: my_cur_sd = round(sqrt(sum(prices * tdiffs) / len(prices)), 2) self.handler.cur_sd = my_cur_sd except: self.test_logger.error("StDev udpate failed - update norm/hft") self.handler.cur_sd = 2 * settings.STOP_OFFSET # in case the stdev failed self.test_logger.error( "update norm parameters completed - update norm/hft") self.last_trim = self.now def __cancel_market_data_request(self): self.conn.cancelMktData(1) time.sleep(1) def spawn(self): print "execution thread spawned" self.handler = ExecutionHandler(self.conn) def start(self, symbols): self.test_logger.info("Started Requests !!!!") self.conn.connect() # Get IB connection object self.__init_stocks_data(symbols) self.test_logger.info("Init Stock") self.test_logger.info("Request Market Data") self.__request_streaming_data(self.conn) self.test_logger.info("Request Position") self.conn.reqPositions() self.test_logger.info("Request Historicals") self.__request_historical_data(self.conn) time.sleep(1) if self.handler.position != 0: self.test_logger.info("Squaring off for a clean start") self.handler.neutralize() try: #self.time_keeper() time.sleep(5) self.test_logger.info("I hope Ihave ohlc now, from hft") print self.ohlc.tail(5) self.test_logger.info("call ML first time - HFT") self.flag = self.ml.call_ml(self.ohlc) self.handler.flag = self.flag # this is stupid time.sleep(3) print "I believe we will " + self.flag self.test_logger.info("Spawn concurrent processes") self.timekeeper = self.executor.submit(self.time_keeper) self.test_logger.info("Time keeper spawned") time.sleep(1) self.parser = self.executor.submit(self.handler.queue_parser) self.test_logger.info("Parser spawned") time.sleep(5) self.update_norm_params() self.test_logger.info("First normalized parameters passed") time.sleep(5) self.test_logger.info("Spawning strategy handler") self.strategy = self.executor.submit(self.handler.trading_loop) self.test_logger.info("Spawning execution handler") self.execution = self.executor.submit(self.handler.order_execution) except (Exception, KeyboardInterrupt): print "Exception:" print "Cancelling...", self.__cancel_market_data_request() print "killing all orders" self.handler.kill_em_all() # self.monitor.close_stream() print "Disconnecting..." time.sleep(5) self.conn.disconnect() time.sleep(1) print "Disconnected." def stop(self): os.remove(os.path.normpath(os.path.join(os.path.curdir, "data.csv"))) os.remove(os.path.normpath(os.path.join(os.path.curdir, "ohlc.csv"))) self.__cancel_market_data_request() #self.monitor.close_stream() print "Disconnecting..." self.conn.disconnect() # self.store.close() def rekindle_execution(self): self.execution = self.executor.submit(self.handler.trading_loop)
class ATS: def __init__(self, host='localhost', port=4001, client_id=101, is_use_gateway=False, evaluation_time_secs=20, resample_interval_secs='30s', moving_window_period=dt.timedelta(hours=1)): self.moving_window_period = moving_window_period self.chart = Chart() self.ib_util = IBUtil() self.strategy_params = StrategyParameters(evaluation_time_secs, resample_interval_secs) self.stocks_data = {} self.symbols = None self.account_code = "" self.prices = None self.trade_qty = 0 self.order_id = 0 self.lock = threading.Lock() self.conn = ibConnection() if is_use_gateway else \ Connection.create(host=host, port=port, clientId=client_id) self.__register_data_handlers(self.__on_tick_event, self.__event_handler) def __perform_trade_logic(self): volatility_ratio = self.strategy_params.get_volatility_ratio() is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1 is_overbought, is_oversold = self.__is_overbought_or_oversold() is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \ (is_down_trend and is_overbought) symbol_a = self.symbols[0] position = self.stocks_data[symbol_a].position is_position_closed, is_short, is_long = \ (position == 0), (position < 0), (position > 0) upnl, rpnl = self.__calculate_pnls() signal_text = \ "BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE" console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \ (dt.datetime.now(), signal_text, position, upnl, rpnl) sys.stdout.write(console_output) sys.stdout.flush() if is_position_closed and is_sell_signal: print "==================================" print "OPEN SHORT POSIITON: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) elif is_position_closed and is_buy_signal: print "==================================" print "OPEN LONG POSIITON: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_short and is_buy_signal: print "==================================" print "CLOSE SHORT POSITION: BUY A SELL B" print "==================================" self.__place_spread_order(self.trade_qty) elif is_long and is_sell_signal: print "==================================" print "CLOSE LONG POSITION: SELL A BUY B" print "==================================" self.__place_spread_order(-self.trade_qty) def __recalculate_strategy_parameters_at_interval(self): if self.strategy_params.is_evaluation_time_elapsed(): self.__calculate_strategy_params() self.strategy_params.set_new_evaluation_time() print '[%s] === Beta re-evaluated ===' % dt.datetime.now() def __calculate_strategy_params(self): [symbol_a, symbol_b] = self.symbols filled_prices = self.prices.fillna(method='ffill') resampled = filled_prices.resample( self.strategy_params.resample_interval_secs, fill_method='ffill')\ .dropna() mean = resampled.mean() beta = mean[symbol_a] / mean[symbol_b] stddevs = resampled.pct_change().dropna().std() volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b] self.strategy_params.add_indicators(beta, volatility_ratio) def __register_data_handlers(self, tick_event_handler, universal_event_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) def __init_stocks_data(self, symbols): self.symbols = symbols self.prices = pd.DataFrame(columns=symbols) for stock_symbol in symbols: contract = self.ib_util.create_stock_contract(stock_symbol) self.stocks_data[stock_symbol] = StockData(contract) def __request_streaming_data(self, ib_conn): for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): ib_conn.reqMktData(index, stock_data.contract, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) time.sleep(1) ib_conn.reqAccountUpdates(True, self.account_code) def __request_historical_data(self, ib_conn): self.lock.acquire() try: for index, (key, stock_data) in enumerate( self.stocks_data.iteritems()): stock_data.is_storing_data = True ib_conn.reqHistoricalData( index, stock_data.contract, time.strftime(datatype.DATE_TIME_FORMAT), datatype.DURATION_1_HR, datatype.BAR_SIZE_5_SEC, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) finally: self.lock.release() def __wait_for_download_completion(self): is_waiting = True while is_waiting: is_waiting = False self.lock.acquire() try: for symbol in self.stocks_data.keys(): if self.stocks_data[symbol].is_storing_data: is_waiting = True finally: self.lock.release() if is_waiting: time.sleep(1) def __place_spread_order(self, qty): [symbol_a, symbol_b] = self.symbols self.__send_order(symbol_a, qty) self.__send_order(symbol_b, -qty) def __send_order(self, symbol, qty): stock_data = self.stocks_data[symbol] order = self.ib_util.create_stock_order(abs(qty), qty > 0) self.conn.placeOrder(self.__generate_order_id(), stock_data.contract, order) stock_data.add_to_position(qty) def __generate_order_id(self): next_order_id = self.order_id self.order_id += 1 return next_order_id def __is_overbought_or_oversold(self): [symbol_a, symbol_b] = self.symbols leg_a_last_price = self.prices[symbol_a].values[-1] leg_b_last_price = self.prices[symbol_b].values[-1] expected_leg_a_price = \ leg_b_last_price * self.strategy_params.get_beta() is_overbought = \ leg_a_last_price < expected_leg_a_price is_oversold = \ leg_a_last_price > expected_leg_a_price return is_overbought, is_oversold def __on_portfolio_update(self, msg): for key, stock_data in self.stocks_data.iteritems(): if stock_data.contract.m_symbol == msg.contract.m_symbol: stock_data.update_position(msg.position, msg.marketPrice, msg.marketValue, msg.averageCost, msg.unrealizedPNL, msg.realizedPNL, msg.accountName) return def __calculate_pnls(self): upnl, rpnl = 0, 0 for key, stock_data in self.stocks_data.iteritems(): upnl += stock_data.unrealized_pnl rpnl += stock_data.realized_pnl return upnl, rpnl def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: self.account_code = msg.accountsList elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID: self.order_id = msg.orderId else: print msg def __on_historical_data(self, msg): print msg ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed(ticker_index) else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self, ticker_index): self.lock.acquire() try: symbol = self.symbols[ticker_index] self.stocks_data[symbol].is_storing_data = False finally: self.lock.release() def __add_historical_data(self, ticker_index, msg): timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT) self.__add_market_data(ticker_index, timestamp, msg.close) def __on_tick_event(self, msg): ticker_id = msg.tickerId field_type = msg.field if field_type == datatype.FIELD_LAST_PRICE: last_price = msg.price self.__add_market_data(ticker_id, dt.datetime.now(), last_price) self.__trim_data_series() if self.strategy_params.is_bootstrap_completed(): self.__recalculate_strategy_parameters_at_interval() self.__perform_trade_logic() self.__update_charts() def __add_market_data(self, ticker_index, timestamp, price): symbol = self.symbols[ticker_index] self.prices.loc[timestamp, symbol] = float(price) self.prices = self.prices.fillna(method='ffill') self.prices.sort_index(inplace=True) def __update_charts(self): if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0: self.chart.display_chart(self.prices, self.strategy_params.indicators) def __trim_data_series(self): cutoff_timestamp = dt.datetime.now() - self.moving_window_period self.prices = self.prices[self.prices.index >= cutoff_timestamp] self.strategy_params.trim_indicators_series(cutoff_timestamp) @staticmethod def __print_elapsed_time(start_time): elapsed_time = time.time() - start_time print "Completed in %.3f seconds." % elapsed_time def __cancel_market_data_request(self): for i, symbol in enumerate(self.symbols): self.conn.cancelMktData(i) time.sleep(1) def start(self, symbols, trade_qty): print "ATS model started." self.trade_qty = trade_qty self.conn.connect() self.__init_stocks_data(symbols) self.__request_streaming_data(self.conn) print "Bootstrapping the model..." start_time = time.time() self.__request_historical_data(self.conn) self.__wait_for_download_completion() self.strategy_params.set_bootstrap_completed() self.__print_elapsed_time(start_time) print "Calculating strategy parameters..." start_time = time.time() self.__calculate_strategy_params() self.__print_elapsed_time(start_time) print "Trading started." try: self.__update_charts() while True: time.sleep(1) except Exception, e: print "Exception:", e print "Cancelling...", self.__cancel_market_data_request() print "Disconnecting..." self.conn.disconnect() time.sleep(1) print "Disconnected."
def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.stocks_data = {} # Dictionary storing StockData objects.REFACTOR self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 #self.lock = Lock() self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10) self.timekeeper = None self.parser = None self.execution = None self.handler = None self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv")) self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")) self.last_trim = None #range/trend flag self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None # self.last_trade = None # self.last_bid = None # self.last_ask = None # self.cur_mean = None # self.cur_sd = None # self.cur_zscore = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) #self.thread = threading.Thread(target=self.spawn()) #self.thread.start() if not self.test: self.handler = ExecutionHandler(self.conn) # #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract("CL", "FUT", "NYMEX", "201606", "USD") else: self.order_template = self.handler.create_contract("CL", "FUT", "NYMEX", "201606", "USD")#todo duplicate with execution handler self.signal = None self.state = None
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.stocks_data = {} # Dictionary storing StockData objects.REFACTOR self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 #self.lock = Lock() self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10) self.timekeeper = None self.parser = None self.execution = None self.handler = None self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv")) self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")) self.last_trim = None #range/trend flag self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None # self.last_trade = None # self.last_bid = None # self.last_ask = None # self.cur_mean = None # self.cur_sd = None # self.cur_zscore = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) #self.thread = threading.Thread(target=self.spawn()) #self.thread.start() if not self.test: self.handler = ExecutionHandler(self.conn) # #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract("CL", "FUT", "NYMEX", "201606", "USD") else: self.order_template = self.handler.create_contract("CL", "FUT", "NYMEX", "201606", "USD")#todo duplicate with execution handler self.signal = None self.state = None def _market_is_open(self): tz_cme = pytz.timezone('America/Chicago') cme_now = self.now.astimezone(tz_cme) if cme_now.weekday() in [0, 1, 2, 3] and ( cme_now >= cme_now.replace(hour=17, minute=0, second=0) or cme_now < cme_now.replace(hour=16, minute=0, second=0)): self.event_market_on.set() elif cme_now.weekday() is 4 and cme_now < cme_now.replace(hour=16, minute=0, second=0): self.event_market_on.set() elif cme_now.weekday() is 6 and cme_now >= cme_now.replace(hour=17, minute=0, second=0): self.event_market_on.set() else: self.event_market_on.clear() # print "market is on:" # print self.event_market_on.is_set() def time_keeper(self): #self.traffic_light.wait() while True: # if self.test == True: print "timekeeper alive" self.now = pytz.timezone('Singapore').localize(dt.datetime.now()) self._market_is_open() # OHLC call if self.last_trim is None: self.last_trim = self.now if self.now > self.last_trim + dt.timedelta(minutes=1): print "call trim from scheduler" self.__request_historical_data(self.conn,initial=False) self.__run_indicators(self.ohlc) print "execution process is alive:" print self.execution.running() print "parser process is alive:" print self.parser.running() print "position is:" print self.handler.position print "I have curmean, cursd, last_trade in execution" print str(self.handler.cur_mean) print str(self.handler.cur_sd) print str(self.handler.last_trade) print "request position:" self.conn.reqPositions() self.update_norm_params() self.last_trim = self.now if self.test: print self.ohlc.tail() time.sleep(5)#TODO horrible, horrible, but can't be bothered with a lock right now # ML Call if self.last_ml_call is None: self.last_ml_call = self.now if self.now > self.last_ml_call + dt.timedelta(minutes=5): print "ML Call" logging.DEBUG("ML Call") logging.DEBUG(str(self.ohlc)) try: self.ml.call_ml(self.ohlc) except: print "ml call failed" print "ML call complete" self.last_ml_call = self.now time.sleep(1) def __register_data_handlers(self, tick_event_handler, universal_event_handler,order_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) self.conn.register(order_handler, ib_message_type.position, ib_message_type.nextValidId, ib_message_type.orderStatus, ib_message_type.openOrder, ib_message_type.error) def __init_stocks_data(self, symbols):#todo brutal hack-through self.symbols = symbols #here we'll store tick and size instead of multiple "symbols" # self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size"))#todomoved to execution # Init price storage if not os.path.exists(self.data_path): self.prices.to_csv(self.data_path) self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage if not os.path.exists(self.ohlc_path): self.ohlc.to_csv(self.ohlc_path) print "checked for csv file" #Now I have only one "symbol" TODO: clean that stuff stock_symbol = self.symbols contract = self.ib_util.create_stock_contract(stock_symbol) self.stocks_data[stock_symbol] = StockData(contract) #this is redundant but required to scaffold/test def create_contract(self, symbol, sec_type, exch, expiry, curr): """Create a Contract object defining what will be purchased, at which exchange and in which currency. symbol - The ticker symbol for the contract sec_type - The security type for the contract ('FUT' = Future) exch - The exchange to carry out the contract on prim_exch - The primary exchange to carry out the contract on curr - The currency in which to purchase the contract""" contract = Contract() contract.m_symbol = symbol contract.m_secType = sec_type contract.m_exchange = exch contract.m_expiry = expiry contract.m_currency = curr return contract def __request_streaming_data(self, ib_conn): # Stream market data. ib_conn.reqMktData(1, self.order_template, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) # time.sleep(5) # Stream account updates DEACTIVATED FOR NOW #ib_conn.reqAccountUpdates(True, self.account_code) def __request_historical_data(self, ib_conn, initial=True): """ the same method can be used for scheduled calls""" # self.ohlc_ok.acquire() if initial: duration = datatype.DURATION_2_HR else: duration = datatype.DURATION_1_MIN ib_conn.reqHistoricalData( 1, self.order_template, time.strftime(datatype.DATE_TIME_FORMAT), duration, datatype.BAR_SIZE_1_MIN, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) def __on_portfolio_update(self, msg): for key, stock_data in self.stocks_data.iteritems(): if stock_data.contract.m_symbol == msg.contract.m_symbol: if dt.datetime.now(self.tz) > self.last_trim + self.moving_window_period: stock_data.update_position(msg.position, msg.marketPrice, msg.marketValue) # , # msg.averageCost, # msg.unrealizedPNL, # msg.realizedPNL, # msg.accountName) return # def __calculate_pnls(self): # upnl, rpnl = 0, 0 # for key, stock_data in self.stocks_data.iteritems(): # upnl += stock_data.unrealized_pnl # rpnl += stock_data.realized_pnl # return upnl, rpnl def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: pass else: print msg def null_handler(self,msg): pass def __on_historical_data(self, msg): ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed() else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self): #self.lock.release() self.last_trim = self.ohlc.index[-1]+self.moving_window_period print "trim time properly set now %s" % self.last_trim print "start position is:" + str(self.handler.position) self.__run_indicators(self.ohlc) self.ohlc.to_csv(self.ohlc_path) # self.ohlc.to_pickle("/Users/maxime_back/Documents/avocado/ohlc.pickle") def __add_historical_data(self, ticker_index, msg): if self.test: print "adding histo line" timestamp = pytz.timezone('Singapore').localize(dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)) self.__add_ohlc_data(timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count) def __add_ohlc_data(self, timestamp, op, hi ,lo,close,vol,cnt): self.ohlc.loc[timestamp, "open"] = float(op) self.ohlc.loc[timestamp, "high"] = float(hi) self.ohlc.loc[timestamp, "low"] = float(lo) self.ohlc.loc[timestamp, "close"] = float(close) self.ohlc.loc[timestamp, "volume"] = float(vol) self.ohlc.loc[timestamp, "count"] = float(cnt) # def __run_indicators(self, ohlc): #hardcoding ML munging parameters now ohlc['returns']=ta.ROC(np.asarray(ohlc['close']).astype(float)) ohlc['sma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 10) ohlc['lma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 120) ohlc['rsi']=ta.RSI(np.asarray(ohlc['close']).astype(float)) ohlc['atr']=ta.ATR(np.asarray(ohlc['high']).astype(float),np.asarray(ohlc['low']).astype(float),np.asarray(ohlc['close']).astype(float),10) ohlc['monday'] = np.where(ohlc.index.weekday == 0,1,0) #bellow is because I don't know how to np.where with multiple conditions ohlc['roll'] = np.where(ohlc.index.month % 3 == 0,1,0) #hackish as balls: # ohlc.index = ohlc.index.tz_localize("Singapore") ohlc["busy"] = np.where(ohlc.index.tz_convert("America/Chicago").hour >= 9,np.where(ohlc.index.tz_convert("America/Chicago").hour <= 14,1,0),0) def update_norm_params(self): # print " updating feeder params for zscore" print "update norm got prices from handler, len:" print len(self.handler.prices) try: prices = self.handler.prices["price"] sgp_tz = pytz.timezone('Singapore') prices.index = prices.index.tz_localize(sgp_tz) prices.to_csv(os.path.join(os.path.curdir,"prices_f_norm.csv")) # print " got prices" prices = prices.dropna() print "sd crapping potential ahead" print "last trim in update norm" print self.last_trim if prices.index.max()-prices.index.min() > dt.timedelta(seconds=60) and self.last_trim is not None: prices = prices[prices.index > self.last_trim]#todo no trim of prices # prices = prices.tail(15)#ik,ik print "sd crap avoided" except: print "update norm FAILED - from update norm" try: if len(prices) !=0: last_price = prices.iloc[-1] self.handler.cur_mean = round(np.mean(prices),2) #logging.debug("updated mean") #print last_price prices = prices.diff() prices = prices.dropna() prices = prices**2 tdiffs = list() for i in range(1,len(prices)): tdiffs.append((prices.index[i]-prices.index[i-1]).total_seconds()) prices = prices.ix[1:] self.handler.cur_sd = round(sqrt(sum(prices * tdiffs)/len(prices)), 2) logging.debug("updated sd") logging.debug(str(self.handler.cur_sd)) # self.cur_zscore = (last_price - self.cur_mean)/self.cur_sd #print(self.cur_zscore) except: print "update normfailed - second halh - from update norm" def __cancel_market_data_request(self): self.conn.cancelMktData(1) time.sleep(1) #recycling zscore spawn to thread the handler def spawn(self): print "execution thread spawned" self.handler = ExecutionHandler(self.conn) def start(self, symbols): print "Start sequence" logging.debug("started requests") self.conn.connect() # Get IB connection object self.__init_stocks_data(symbols) print "init stock" print "request mkt data" self.__request_streaming_data(self.conn) print "request position" self.conn.reqPositions() print "request historicals" self.__request_historical_data(self.conn) time.sleep(1) if self.handler.position !=0: print "squaring position for a clean start" self.handler.neutralize() try: print "getting ohlc data now" #self.time_keeper() time.sleep(5) print "pray I have them now" print self.ohlc.tail(5) print "calling ML for the first time" self.flag = self.ml.call_ml(self.ohlc) self.handler.flag = self.flag# this is stupid time.sleep(3) print "I believe we will "+ self.flag # if self.test: # print self.ohlc # time.sleep(60) # print "now calling for update" # self.__request_historical_data(self.conn,initial=False) # print self.ohlc print "spawn concurrent processes" self.timekeeper = self.executor.submit(self.time_keeper) print "timekeeper spawned" time.sleep(1) self.parser = self.executor.submit(self.handler.queue_parser) print "parser spawned" time.sleep(5) self.update_norm_params() print "sd/mean: passed" time.sleep(5) print "handler spawned" self.execution = self.executor.submit(self.handler.trading_loop) except (Exception, KeyboardInterrupt): print "Exception:" print "Cancelling...", self.__cancel_market_data_request() print "killing all orders" self.handler.kill_em_all() # self.monitor.close_stream() print "Disconnecting..." time.sleep(5) self.conn.disconnect() time.sleep(1) print "Disconnected." def stop(self): os.remove(os.path.normpath(os.path.join(os.path.curdir,"data.csv"))) os.remove(os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))) self.__cancel_market_data_request() #self.monitor.close_stream() print "Disconnecting..." self.conn.disconnect() # self.store.close() def spawn_test(self): self.traffic_light.wait() print "f**k spawns"
def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): logging.basicConfig(format='%(asctime)s %(message)s') self.test_logger = logging.getLogger('hftModelLogger') self.test_logger.setLevel(logging.INFO) self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6) self.timekeeper = None self.parser = None self.execution = None self.strategy = None self.handler = None self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv")) self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")) self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None self.cur_mean = None self.cur_sd = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) if not self.test: self.handler = ExecutionHandler(self.conn) #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) else: self.order_template = self.handler.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) self.signal = None self.state = None
class HFTModel: def __init__(self, host='localhost', port=4001, client_id=130, is_use_gateway=False, moving_window_period=dt.timedelta(seconds=60), test=False): logging.basicConfig(format='%(asctime)s %(message)s') self.test_logger = logging.getLogger('hftModelLogger') self.test_logger.setLevel(logging.INFO) self.test = test self.tz = pytz.timezone('Singapore') self.moving_window_period = moving_window_period self.ib_util = IBUtil() self.symbols = None # List of current symbols self.account_code = "" self.prices = None # Store last prices in a DataFrame self.ohlc = None # I need another store for minute data (I think) self.buffer = list() self.trade_qty = 0 self.traffic_light = Event() self.ohlc_ok = Lock() self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6) self.timekeeper = None self.parser = None self.execution = None self.strategy = None self.handler = None self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv")) self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")) self.flag = None self.event_market_on = Event() self.ml = MLcall() self.last_trim = None self.last_ml_call = None self.cur_mean = None self.cur_sd = None # Use ibConnection() for TWS, or create connection for API Gateway self.conn = Connection.create(host=host, port=port, clientId=client_id) if not self.test: self.handler = ExecutionHandler(self.conn) #third handler should register properly si Dieu veut if self.test: self.__register_data_handlers(self.null_handler, self.__event_handler, self.null_handler) else: self.__register_data_handlers(self.handler.on_tick_event, self.__event_handler, self.handler._reply_handler) if self.test: self.order_template = self.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) else: self.order_template = self.handler.create_contract(settings.SYMBOL, settings.SECURITY, settings.EXCHANGE, settings.EXPIRY, settings.CURRENCY) self.signal = None self.state = None def _market_is_open(self): tz_cme = pytz.timezone('America/Chicago') cme_now = self.now.astimezone(tz_cme) if cme_now.weekday() in [0, 1, 2, 3] and ( cme_now >= cme_now.replace(hour=17, minute=0, second=0) or cme_now < cme_now.replace(hour=16, minute=0, second=0)): self.event_market_on.set() elif cme_now.weekday() is 4 and cme_now < cme_now.replace(hour=16, minute=0, second=0): self.event_market_on.set() elif cme_now.weekday() is 6 and cme_now >= cme_now.replace(hour=17, minute=0, second=0): self.event_market_on.set() else: self.event_market_on.clear() # print "market is on:" # print self.event_market_on.is_set() def time_keeper(self): while True: print "executing and trading flag set?" print self.handler.executing.is_set() print self.handler.trading.is_set() self.now = pytz.timezone('Singapore').localize(dt.datetime.now()) self._market_is_open() # OHLC call if self.last_trim is None: self.last_trim = self.now if self.now > self.last_trim + dt.timedelta(minutes=1): if self.parser.running(): self.test_logger.error("parser thread is alive - timekeeper") if self.strategy.running(): self.test_logger.error("strategy thread is alive - timekeeper") if self.execution.running(): self.test_logger.error("execution thread is alive - timekeeper") else: self.test_logger.error("!!!execution thread is DEAD- timekeeper") self.test_logger.error("timekeeper minute call") try: self.__request_historical_data(self.conn,initial=False) except: self.test_logger.error("req of update historicals failed - timekeep/hft") try: self.__run_indicators(self.ohlc) except: self.test_logger.error("runnning indicators failed - timekeep/hft") self.conn.reqPositions() self.update_norm_params() if self.test: print self.ohlc.tail() time.sleep(5)#TODO horrible, horrible, but can't be bothered with a lock right now # ML Call if self.last_ml_call is None: self.last_ml_call = self.now if self.now > self.last_ml_call + dt.timedelta(minutes=5): #logging.DEBUG("ML Call") #logging.DEBUG(str(self.ohlc)) try: self.flag = self.ml.call_ml(self.ohlc) self.handler.flag = self.flag # this is stupid todo why why why self.test_logger.error("I believe we will " + self.handler.flag) self.last_ml_call = self.now except: self.test_logger.error("!!! Call ML failed - timekeeper/hft") time.sleep(1) def __register_data_handlers(self, tick_event_handler, universal_event_handler,order_handler): self.conn.registerAll(universal_event_handler) self.conn.unregister(universal_event_handler, ib_message_type.tickSize, ib_message_type.tickPrice, ib_message_type.tickString, ib_message_type.tickGeneric, ib_message_type.tickOptionComputation) self.conn.register(tick_event_handler, ib_message_type.tickPrice, ib_message_type.tickSize) self.conn.register(order_handler, ib_message_type.position, ib_message_type.nextValidId, ib_message_type.orderStatus, ib_message_type.openOrder, ib_message_type.error) def __init_stocks_data(self, symbols):#todo brutal hack-through self.symbols = symbols #here we'll store tick and size instead of multiple "symbols" self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size"))#todomoved to execution # Init price storage if not os.path.exists(self.data_path): self.prices.to_csv(self.data_path) self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage if not os.path.exists(self.ohlc_path): self.ohlc.to_csv(self.ohlc_path) print "checked for csv file" #Now I have only one "symbol" TODO: clean that stuff stock_symbol = self.symbols contract = self.ib_util.create_stock_contract(stock_symbol) #self.stocks_data[stock_symbol] = StockData(contract) #this is redundant but required to scaffold/test def create_contract(self, symbol, sec_type, exch, expiry, curr): """Create a Contract object defining what will be purchased, at which exchange and in which currency. symbol - The ticker symbol for the contract sec_type - The security type for the contract ('FUT' = Future) exch - The exchange to carry out the contract on prim_exch - The primary exchange to carry out the contract on curr - The currency in which to purchase the contract""" contract = Contract() contract.m_symbol = symbol contract.m_secType = sec_type contract.m_exchange = exch contract.m_expiry = expiry contract.m_currency = curr return contract def __request_streaming_data(self, ib_conn): # Stream market data. ib_conn.reqMktData(1, self.order_template, datatype.GENERIC_TICKS_NONE, datatype.SNAPSHOT_NONE) # time.sleep(5) def __request_historical_data(self, ib_conn, initial=True): """ the same method can be used for scheduled calls""" # self.ohlc_ok.acquire() if initial: duration = datatype.DURATION_2_HR else: duration = datatype.DURATION_1_MIN ib_conn.reqHistoricalData( 1, self.order_template, time.strftime(datatype.DATE_TIME_FORMAT), duration, datatype.BAR_SIZE_1_MIN, datatype.WHAT_TO_SHOW_TRADES, datatype.RTH_ALL, datatype.DATEFORMAT_STRING) time.sleep(1) def __event_handler(self, msg): if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA: self.__on_historical_data(msg) elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO: pass #self.__on_portfolio_update(msg) elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS: pass else: print msg def null_handler(self,msg): pass def __on_historical_data(self, msg): ticker_index = msg.reqId if msg.WAP == -1: self.__on_historical_data_completed() else: self.__add_historical_data(ticker_index, msg) def __on_historical_data_completed(self): #self.lock.release() self.last_trim = self.ohlc.index[-1]+self.moving_window_period print "trim time properly set now %s" % self.last_trim print "start position is:" + str(self.handler.position) self.__run_indicators(self.ohlc) self.ohlc.to_csv(self.ohlc_path) def __add_historical_data(self, ticker_index, msg): if self.test: print "adding histo line" timestamp = pytz.timezone('Singapore').localize(dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)) self.__add_ohlc_data(timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count) def __add_ohlc_data(self, timestamp, op, hi ,lo,close,vol,cnt): self.ohlc.loc[timestamp, "open"] = float(op) self.ohlc.loc[timestamp, "high"] = float(hi) self.ohlc.loc[timestamp, "low"] = float(lo) self.ohlc.loc[timestamp, "close"] = float(close) self.ohlc.loc[timestamp, "volume"] = float(vol) self.ohlc.loc[timestamp, "count"] = float(cnt) # def __run_indicators(self, ohlc): #hardcoding ML munging parameters now ohlc['returns']=ta.ROC(np.asarray(ohlc['close']).astype(float)) ohlc['sma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 10) ohlc['lma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 120) ohlc['rsi']=ta.RSI(np.asarray(ohlc['close']).astype(float)) ohlc['atr']=ta.ATR(np.asarray(ohlc['high']).astype(float),np.asarray(ohlc['low']).astype(float),np.asarray(ohlc['close']).astype(float),10) ohlc['monday'] = np.where(ohlc.index.weekday == 0,1,0) #bellow is because I don't know how to np.where with multiple conditions ohlc['roll'] = np.where(ohlc.index.month % 3 == 0,1,0) #hackish as balls: # ohlc.index = ohlc.index.tz_localize("Singapore") ohlc["busy"] = np.where(ohlc.index.tz_convert("America/Chicago").hour >= 9,np.where(ohlc.index.tz_convert("America/Chicago").hour <= 14,1,0),0) def update_norm_params(self): try: prices = self.handler.prices["price"] except: self.test_logger.error("getting price from handler failed - update norm/hft") sgp_tz = pytz.timezone('Singapore') try: prices.index = prices.index.tz_localize(sgp_tz) #prices.to_pick (os.path.join(os.path.curdir,"prices_f_norm.csv")) except: self.test_logger.error("prices localization failed - update norm/hft") # try: prices = prices.dropna() except: self.test_logger.error("dropna failed - update norm/hft") try: if prices.index.max()-prices.index.min() > dt.timedelta(seconds=60) and self.last_trim is not None: prices = prices[prices.index > prices.index.max()-dt.timedelta(seconds=60)]#todo no trim of prices as of now print "trimmed prices" print prices #prices.to_pickle(os.path.join(os.path.curdir(), "prices.pkl")) except: self.test_logger.error("minute trim of prices failed - update norm/hft") if len(prices) !=0: last_price = prices.iloc[-1] try: my_cur_mean = round(np.mean(prices),2) self.handler.cur_mean = my_cur_mean except: self.test_logger.error("mean update failed- update norm/hft") try: prices = prices.diff() prices = prices.dropna() prices = prices**2 except: self.test_logger.error("diff/square failed- update norm/hft") tdiffs = list() try: for i in range(1,len(prices)): tdiffs.append((prices.index[i]-prices.index[i-1]).total_seconds()) prices = prices.ix[1:] print prices except: self.test_logger.error("time diffs creation failed - update norm/hft") try: my_cur_sd = round(sqrt(sum(prices * tdiffs)/len(prices)), 2) self.handler.cur_sd = my_cur_sd except: self.test_logger.error("StDev udpate failed - update norm/hft") self.handler.cur_sd = 2 * settings.STOP_OFFSET # in case the stdev failed self.test_logger.error("update norm parameters completed - update norm/hft") self.last_trim = self.now def __cancel_market_data_request(self): self.conn.cancelMktData(1) time.sleep(1) def spawn(self): print "execution thread spawned" self.handler = ExecutionHandler(self.conn) def start(self, symbols): self.test_logger.info("Started Requests !!!!") self.conn.connect() # Get IB connection object self.__init_stocks_data(symbols) self.test_logger.info("Init Stock") self.test_logger.info("Request Market Data") self.__request_streaming_data(self.conn) self.test_logger.info("Request Position") self.conn.reqPositions() self.test_logger.info("Request Historicals") self.__request_historical_data(self.conn) time.sleep(1) if self.handler.position !=0: self.test_logger.info("Squaring off for a clean start") self.handler.neutralize() try: #self.time_keeper() time.sleep(5) self.test_logger.info("I hope Ihave ohlc now, from hft") print self.ohlc.tail(5) self.test_logger.info("call ML first time - HFT") self.flag = self.ml.call_ml(self.ohlc) self.handler.flag = self.flag # this is stupid time.sleep(3) print "I believe we will "+ self.flag self.test_logger.info("Spawn concurrent processes") self.timekeeper = self.executor.submit(self.time_keeper) self.test_logger.info("Time keeper spawned") time.sleep(1) self.parser = self.executor.submit(self.handler.queue_parser) self.test_logger.info("Parser spawned") time.sleep(5) self.update_norm_params() self.test_logger.info("First normalized parameters passed") time.sleep(5) self.test_logger.info("Spawning strategy handler") self.strategy = self.executor.submit(self.handler.trading_loop) self.test_logger.info("Spawning execution handler") self.execution = self.executor.submit(self.handler.order_execution) except (Exception, KeyboardInterrupt): print "Exception:" print "Cancelling...", self.__cancel_market_data_request() print "killing all orders" self.handler.kill_em_all() # self.monitor.close_stream() print "Disconnecting..." time.sleep(5) self.conn.disconnect() time.sleep(1) print "Disconnected." def stop(self): os.remove(os.path.normpath(os.path.join(os.path.curdir,"data.csv"))) os.remove(os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))) self.__cancel_market_data_request() #self.monitor.close_stream() print "Disconnecting..." self.conn.disconnect() # self.store.close() def rekindle_execution(self): self.execution = self.executor.submit(self.handler.trading_loop)