def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(seconds=60)):
self.moving_window_period = moving_window_period
# self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
# self.strategy_params = StrategyParameters(evaluation_time_secs,
# resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
#addition for hdf store
self.data_path = os.path.normpath("/Users/maxime_back/Documents/avocado/data.csv")
self.ohlc_path = os.path.normpath("/Users/maxime_back/Documents/avocado/ohlc.csv")
self.last_trim = dt.datetime.now()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {}
self.symbols = None
self.account_code = ""
self.prices = None
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
class HFTModel:
def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __perform_trade_logic(self):
"""
This part is the 'secret-sauce' where actual trades takes place.
My take is that great experience, good portfolio construction,
and together with robust backtesting will make your strategy viable.
GOOD PORTFOLIO CONTRUCTION CAN SAVE YOU FROM BAD RESEARCH,
BUT BAD PORTFOLIO CONSTRUCTION CANNOT SAVE YOU FROM GREAT RESEARCH
This trade logic uses volatility ratio and beta as our indicators.
- volatility ratio > 1 :: uptrend, volatility ratio < 1 :: downtrend
- beta is calculated as: mean(price A) / mean(price B)
We use the assumption that prive levels will mean-revert.
Expected price A = beta x price B
Consider other methods of identifying our trade logic:
- current trend
- current regime
- detect structural breaks
"""
volatility_ratio = self.strategy_params.get_volatility_ratio()
is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1
is_overbought, is_oversold = self.__is_overbought_or_oversold()
# Our final trade signals
is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \
(is_down_trend and is_overbought)
# Use account position details
symbol_a = self.symbols[0]
position = self.stocks_data[symbol_a].position
is_position_closed, is_short, is_long = \
(position == 0), (position < 0), (position > 0)
upnl, rpnl = self.__calculate_pnls()
# Display to terminal dynamically
signal_text = \
"BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE"
console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \
(dt.datetime.now(), signal_text, position, upnl, rpnl)
sys.stdout.write(console_output)
sys.stdout.flush()
if is_position_closed and is_sell_signal:
print "=================================="
print "OPEN SHORT POSIITON: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
elif is_position_closed and is_buy_signal:
print "=================================="
print "OPEN LONG POSIITON: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_short and is_buy_signal:
print "=================================="
print "CLOSE SHORT POSITION: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_long and is_sell_signal:
print "=================================="
print "CLOSE LONG POSITION: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
def __recalculate_strategy_parameters_at_interval(self):
"""
Consider re-evaluation of parameters on:
- regime shifts
- structural breaks
"""
if self.strategy_params.is_evaluation_time_elapsed():
self.__calculate_strategy_params()
self.strategy_params.set_new_evaluation_time()
print '[%s] === Beta re-evaluated ===' % dt.datetime.now()
def __calculate_strategy_params(self):
"""
Here, we are calculating beta and volatility ratio
for our signal indicators.
Consider calculating other statistics here:
- stddevs of errs
- correlations
- co-integration
"""
[symbol_a, symbol_b] = self.symbols
filled_prices = self.prices.fillna(method='ffill')
resampled = filled_prices.resample(
self.strategy_params.resample_interval_secs, fill_method='ffill')\
.dropna()
mean = resampled.mean()
beta = mean[symbol_a] / mean[symbol_b]
stddevs = resampled.pct_change().dropna().std()
volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b]
self.strategy_params.add_indicators(beta, volatility_ratio)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
def __init_stocks_data(self, symbols):
self.symbols = symbols
self.prices = pd.DataFrame(columns=symbols) # Init price storage
for stock_symbol in symbols:
contract = self.ib_util.create_stock_contract(stock_symbol)
self.stocks_data[stock_symbol] = StockData(contract)
def __request_streaming_data(self, ib_conn):
# Stream market data
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
ib_conn.reqMktData(index,
stock_data.contract,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
time.sleep(1)
# Stream account updates
ib_conn.reqAccountUpdates(True, self.account_code)
def __request_historical_data(self, ib_conn):
self.lock.acquire()
try:
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
stock_data.is_storing_data = True
ib_conn.reqHistoricalData(
index,
stock_data.contract,
time.strftime(datatype.DATE_TIME_FORMAT),
datatype.DURATION_1_HR,
datatype.BAR_SIZE_5_SEC,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
finally:
self.lock.release()
def __wait_for_download_completion(self):
is_waiting = True
while is_waiting:
is_waiting = False
self.lock.acquire()
try:
for symbol in self.stocks_data.keys():
if self.stocks_data[symbol].is_storing_data:
is_waiting = True
finally:
self.lock.release()
if is_waiting:
time.sleep(1)
def __place_spread_order(self, qty):
[symbol_a, symbol_b] = self.symbols
self.__send_order(symbol_a, qty)
self.__send_order(symbol_b, -qty)
def __send_order(self, symbol, qty):
stock_data = self.stocks_data[symbol]
order = self.ib_util.create_stock_order(abs(qty), qty > 0)
self.conn.placeOrder(self.__generate_order_id(),
stock_data.contract,
order)
stock_data.add_to_position(qty)
def __generate_order_id(self):
next_order_id = self.order_id
self.order_id += 1
return next_order_id
def __is_overbought_or_oversold(self):
[symbol_a, symbol_b] = self.symbols
leg_a_last_price = self.prices[symbol_a].values[-1]
leg_b_last_price = self.prices[symbol_b].values[-1]
expected_leg_a_price = \
leg_b_last_price * self.strategy_params.get_beta()
is_overbought = \
leg_a_last_price < expected_leg_a_price # Cheaper than expected
is_oversold = \
leg_a_last_price > expected_leg_a_price # Higher than expected
return is_overbought, is_oversold
def __on_portfolio_update(self, msg):
for key, stock_data in self.stocks_data.iteritems():
if stock_data.contract.m_symbol == msg.contract.m_symbol:
stock_data.update_position(msg.position,
msg.marketPrice,
msg.marketValue,
msg.averageCost,
msg.unrealizedPNL,
msg.realizedPNL,
msg.accountName)
return
def __calculate_pnls(self):
upnl, rpnl = 0, 0
for key, stock_data in self.stocks_data.iteritems():
upnl += stock_data.unrealized_pnl
rpnl += stock_data.realized_pnl
return upnl, rpnl
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
self.account_code = msg.accountsList
elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID:
self.order_id = msg.orderId
else:
print msg
def __on_historical_data(self, msg):
print msg
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed(ticker_index)
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self, ticker_index):
self.lock.acquire()
try:
symbol = self.symbols[ticker_index]
self.stocks_data[symbol].is_storing_data = False
finally:
self.lock.release()
def __add_historical_data(self, ticker_index, msg):
timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)
self.__add_market_data(ticker_index, timestamp, msg.close)
def __on_tick_event(self, msg):
ticker_id = msg.tickerId
field_type = msg.field
# Store information from last traded price
if field_type == datatype.FIELD_LAST_PRICE:
last_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), last_price)
self.__trim_data_series()
# Post-bootstrap - make trading decisions
if self.strategy_params.is_bootstrap_completed():
self.__recalculate_strategy_parameters_at_interval()
self.__perform_trade_logic()
self.__update_charts()
def __add_market_data(self, ticker_index, timestamp, price):
symbol = self.symbols[ticker_index]
self.prices.loc[timestamp, symbol] = float(price)
self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
def __update_charts(self):
if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0:
self.chart.display_chart(self.prices,
self.strategy_params.indicators)
def __trim_data_series(self):
cutoff_timestamp = dt.datetime.now() - self.moving_window_period
self.prices = self.prices[self.prices.index >= cutoff_timestamp]
self.strategy_params.trim_indicators_series(cutoff_timestamp)
@staticmethod
def __print_elapsed_time(start_time):
elapsed_time = time.time() - start_time
print "Completed in %.3f seconds." % elapsed_time
def __cancel_market_data_request(self):
for i, symbol in enumerate(self.symbols):
self.conn.cancelMktData(i)
time.sleep(1)
def start(self, symbols, trade_qty):
print "HFT model started."
self.trade_qty = trade_qty
self.conn.connect() # Get IB connection object
self.__init_stocks_data(symbols)
self.__request_streaming_data(self.conn)
print "Bootstrapping the model..."
start_time = time.time()
self.__request_historical_data(self.conn)
self.__wait_for_download_completion()
self.strategy_params.set_bootstrap_completed()
self.__print_elapsed_time(start_time)
print "Calculating strategy parameters..."
start_time = time.time()
self.__calculate_strategy_params()
self.__print_elapsed_time(start_time)
print "Trading started."
try:
self.__update_charts()
while True:
time.sleep(1)
except Exception, e:
print "Exception:", e
print "Cancelling...",
self.__cancel_market_data_request()
print "Disconnecting..."
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
class HFTModel:
def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __perform_trade_logic(self):
"""
This part is the 'secret-sauce' where actual trades takes place.
My take is that great experience, good portfolio construction,
and together with robust backtesting will make your strategy viable.
GOOD PORTFOLIO CONTRUCTION CAN SAVE YOU FROM BAD RESEARCH,
BUT BAD PORTFOLIO CONSTRUCTION CANNOT SAVE YOU FROM GREAT RESEARCH
This trade logic uses volatility ratio and beta as our indicators.
- volatility ratio > 1 :: uptrend, volatility ratio < 1 :: downtrend
- beta is calculated as: mean(price A) / mean(price B)
We use the assumption that prive levels will mean-revert.
Expected price A = beta x price B
Consider other methods of identifying our trade logic:
- current trend
- current regime
- detect structural breaks
"""
volatility_ratio = self.strategy_params.get_volatility_ratio()
is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1
is_overbought, is_oversold = self.__is_overbought_or_oversold()
# Our final trade signals
is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \
(is_down_trend and is_overbought)
# Use account position details
symbol_a = self.symbols[0]
position = self.stocks_data[symbol_a].position
is_position_closed, is_short, is_long = \
(position == 0), (position < 0), (position > 0)
upnl, rpnl = self.__calculate_pnls()
# Display to terminal dynamically
signal_text = \
"BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE"
console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \
(dt.datetime.now(), signal_text, position, upnl, rpnl)
sys.stdout.write(console_output)
sys.stdout.flush()
if is_position_closed and is_sell_signal:
print "=================================="
print "OPEN SHORT POSIITON: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
elif is_position_closed and is_buy_signal:
print "=================================="
print "OPEN LONG POSIITON: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_short and is_buy_signal:
print "=================================="
print "CLOSE SHORT POSITION: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_long and is_sell_signal:
print "=================================="
print "CLOSE LONG POSITION: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
def __recalculate_strategy_parameters_at_interval(self):
"""
Consider re-evaluation of parameters on:
- regime shifts
- structural breaks
"""
if self.strategy_params.is_evaluation_time_elapsed():
self.__calculate_strategy_params()
self.strategy_params.set_new_evaluation_time()
print '[%s] === Beta re-evaluated ===' % dt.datetime.now()
def __calculate_strategy_params(self):
"""
Here, we are calculating beta and volatility ratio
for our signal indicators.
Consider calculating other statistics here:
- stddevs of errs
- correlations
- co-integration
"""
[symbol_a, symbol_b] = self.symbols
filled_prices = self.prices.fillna(method='ffill')
resampled = filled_prices.resample(
self.strategy_params.resample_interval_secs, fill_method='ffill')\
.dropna()
mean = resampled.mean()
beta = mean[symbol_a] / mean[symbol_b]
stddevs = resampled.pct_change().dropna().std()
volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b]
self.strategy_params.add_indicators(beta, volatility_ratio)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
def __init_stocks_data(self, symbols):
self.symbols = symbols
self.prices = pd.DataFrame(columns=symbols) # Init price storage
for stock_symbol in symbols:
contract = self.ib_util.create_stock_contract(stock_symbol)
self.stocks_data[stock_symbol] = StockData(contract)
def __request_streaming_data(self, ib_conn):
# Stream market data
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
ib_conn.reqMktData(index,
stock_data.contract,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
time.sleep(1)
# Stream account updates
ib_conn.reqAccountUpdates(True, self.account_code)
def __request_historical_data(self, ib_conn):
self.lock.acquire()
try:
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
stock_data.is_storing_data = True
ib_conn.reqHistoricalData(
index,
stock_data.contract,
time.strftime(datatype.DATE_TIME_FORMAT),
datatype.DURATION_1_HR,
datatype.BAR_SIZE_5_SEC,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
finally:
self.lock.release()
def __wait_for_download_completion(self):
is_waiting = True
while is_waiting:
is_waiting = False
self.lock.acquire()
try:
for symbol in self.stocks_data.keys():
if self.stocks_data[symbol].is_storing_data:
is_waiting = True
finally:
self.lock.release()
if is_waiting:
time.sleep(1)
def __place_spread_order(self, qty):
[symbol_a, symbol_b, symbol_c] = self.symbols
self.__send_order(symbol_a, qty)
time.sleep(3)
self.__send_order(symbol_b, qty)
time.sleep(3)
self.__send_order(symbol_c, -qty)
time.sleep(3)
def __send_order(self, symbol, qty):
stock_data = self.stocks_data[symbol]
order = self.ib_util.create_stock_order(abs(qty), qty > 0)
self.conn.placeOrder(self.__generate_order_id(),
stock_data.contract,
order)
stock_data.add_to_position(qty)
def __generate_order_id(self):
next_order_id = self.order_id
self.order_id += 1
return next_order_id
def __is_overbought_or_oversold(self):
[symbol_a, symbol_b] = self.symbols
leg_a_last_price = self.prices[symbol_a].values[-1]
leg_b_last_price = self.prices[symbol_b].values[-1]
expected_leg_a_price = \
leg_b_last_price * self.strategy_params.get_beta()
is_overbought = \
leg_a_last_price < expected_leg_a_price # Cheaper than expected
is_oversold = \
leg_a_last_price > expected_leg_a_price # Higher than expected
return is_overbought, is_oversold
def __on_portfolio_update(self, msg):
for key, stock_data in self.stocks_data.iteritems():
if stock_data.contract.m_symbol == msg.contract.m_symbol:
stock_data.update_position(msg.position,
msg.marketPrice,
msg.marketValue,
msg.averageCost,
msg.unrealizedPNL,
msg.realizedPNL,
msg.accountName)
return
def __calculate_pnls(self):
upnl, rpnl = 0, 0
for key, stock_data in self.stocks_data.iteritems():
upnl += stock_data.unrealized_pnl
rpnl += stock_data.realized_pnl
return upnl, rpnl
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
self.account_code = msg.accountsList
elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID:
self.order_id = msg.orderId
else:
print msg
def __on_historical_data(self, msg):
print msg
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed(ticker_index)
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self, ticker_index):
self.lock.acquire()
try:
symbol = self.symbols[ticker_index]
self.stocks_data[symbol].is_storing_data = False
finally:
self.lock.release()
def __add_historical_data(self, ticker_index, msg):
timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)
self.__add_market_data(ticker_index, timestamp, msg.close)
def __on_tick_event(self, msg):
ticker_id = msg.tickerId
field_type = msg.field
# Store information from last traded price
if field_type == datatype.FIELD_LAST_PRICE:
last_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), last_price)
self.__trim_data_series()
# Post-bootstrap - make trading decisions
if self.strategy_params.is_bootstrap_completed():
self.__recalculate_strategy_parameters_at_interval()
self.__perform_trade_logic()
self.__update_charts()
def __add_market_data(self, ticker_index, timestamp, price):
symbol = self.symbols[ticker_index]
self.prices.loc[timestamp, symbol] = float(price)
self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
def __update_charts(self):
if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0:
self.chart.display_chart(self.prices,
self.strategy_params.indicators)
def __trim_data_series(self):
cutoff_timestamp = dt.datetime.now() - self.moving_window_period
self.prices = self.prices[self.prices.index >= cutoff_timestamp]
self.strategy_params.trim_indicators_series(cutoff_timestamp)
@staticmethod
def __print_elapsed_time(start_time):
elapsed_time = time.time() - start_time
print "Completed in %.3f seconds." % elapsed_time
def __cancel_market_data_request(self):
for i, symbol in enumerate(self.symbols):
self.conn.cancelMktData(i)
time.sleep(1)
def start(self, symbols, trade_qty):
print "HFT model started."
self.trade_qty = trade_qty
self.conn.connect() # Get IB connection object
self.__init_stocks_data(symbols)
self.__request_streaming_data(self.conn)
print "Bootstrapping the model..."
start_time = time.time()
self.__request_historical_data(self.conn)
self.__wait_for_download_completion()
self.strategy_params.set_bootstrap_completed()
self.__print_elapsed_time(start_time)
print "Calculating strategy parameters..."
start_time = time.time()
self.__calculate_strategy_params()
self.__print_elapsed_time(start_time)
print "Trading started."
try:
self.__update_charts()
while True:
time.sleep(1)
except Exception, e:
print "Exception:", e
print "Cancelling...",
self.__cancel_market_data_request()
print "Disconnecting..."
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
class HFTModel:
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(seconds=60)):
self.moving_window_period = moving_window_period
# self.chart = Chart()
self.ib_util = IBUtil()
# Store parameters for this model
# self.strategy_params = StrategyParameters(evaluation_time_secs,
# resample_interval_secs)
self.stocks_data = {} # Dictionary storing StockData objects.
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
#addition for hdf store
self.data_path = os.path.normpath("/Users/maxime_back/Documents/avocado/data.csv")
self.ohlc_path = os.path.normpath("/Users/maxime_back/Documents/avocado/ohlc.csv")
self.last_trim = dt.datetime.now()
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
def __init_stocks_data(self, symbols):
self.symbols = symbols
#here we'll store tick and size instead of multiple "symbols"
self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size")) # Init price storage
if not os.path.exists(self.data_path):
self.prices.to_csv(self.data_path)
self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage
if not os.path.exists(self.ohlc_path):
self.ohlc.to_csv(self.ohlc_path)
print "checked for csv file"
#Now I have only one "symbol"
stock_symbol = self.symbols
contract = self.ib_util.create_stock_contract(stock_symbol)
self.stocks_data[stock_symbol] = StockData(contract)
print "contracts initated"
def __request_streaming_data(self, ib_conn):
# Stream market data. Of note: this enumerate can probably be simplified
#cannt be bothered for now
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
ib_conn.reqMktData(index,
stock_data.contract,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
time.sleep(5)
# Stream account updates DEACTIVATED FOR NOW
# ib_conn.reqAccountUpdates(True, self.account_code)
def __request_historical_data(self, ib_conn):
self.lock.acquire()
try:
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
stock_data.is_storing_data = True
ib_conn.reqHistoricalData(
index,
stock_data.contract,
time.strftime(datatype.DATE_TIME_FORMAT),
datatype.DURATION_1_DAY,
datatype.BAR_SIZE_1_MIN,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
finally:
self.lock.release()
self.last_trim = dt.datetime.now()
# def __wait_for_download_completion(self):
# is_waiting = True
# while is_waiting:
# is_waiting = False
#
# self.lock.acquire()
# try:
# for symbol in self.stocks_data.keys():
# if self.stocks_data[symbol].is_storing_data:
# is_waiting = True
# finally:
# self.lock.release()
#
# if is_waiting:
# time.sleep(1)
# def __on_portfolio_update(self, msg):
# for key, stock_data in self.stocks_data.iteritems():
# if stock_data.contract.m_symbol == msg.contract.m_symbol:
# stock_data.update_position(msg.position,
# msg.marketPrice,
# msg.marketValue,
# msg.averageCost,
# msg.unrealizedPNL,
# msg.realizedPNL,
# msg.accountName)
# return
#
# def __calculate_pnls(self):
# upnl, rpnl = 0, 0
# for key, stock_data in self.stocks_data.iteritems():
# upnl += stock_data.unrealized_pnl
# rpnl += stock_data.realized_pnl
# return upnl, rpnl
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
pass
# self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
pass
# self.account_code = msg.accountsList
elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID:
self.order_id = msg.orderId
else:
print msg
def __on_historical_data(self, msg):
print msg
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed(ticker_index)
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self, ticker_index):
self.lock.acquire()
try:
symbol = self.symbols#[ticker_index]
self.stocks_data[symbol].is_storing_data = False
finally:
self.lock.release()
def __add_historical_data(self, ticker_index, msg):
timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)
self.__add_ohlc_data(ticker_index, timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count)
def __add_ohlc_data(self, ticker_index, timestamp, op, hi ,lo,close,vol,cnt ):
self.ohlc.loc[timestamp, "open"] = float(op)
self.ohlc.loc[timestamp, "high"] = float(hi)
self.ohlc.loc[timestamp, "low"] = float(lo)
self.ohlc.loc[timestamp, "close"] = float(close)
self.ohlc.loc[timestamp, "volume"] = float(vol)
self.ohlc.loc[timestamp, "count"] = float(cnt)
def __on_tick_event(self, msg):
ticker_id = msg.tickerId
field_type = msg.field
# print field_type
# Store information from last traded price
if field_type == datatype.FIELD_LAST_PRICE:
last_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), last_price, 1)
if field_type == datatype.FIELD_LAST_SIZE:
last_size = msg.size
self.__add_market_data(ticker_id, dt.datetime.now(), last_size, 2)
if field_type == datatype.FIELD_ASK_PRICE:
ask_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), ask_price, 3)
if field_type == datatype.FIELD_ASK_SIZE:
ask_size = msg.size
self.__add_market_data(ticker_id, dt.datetime.now(), ask_size, 4)
if field_type == datatype.FIELD_BID_PRICE:
bid_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), bid_price, 5)
if field_type == datatype.FIELD_BID_SIZE:
bid_size = msg.size
self.__add_market_data(ticker_id, dt.datetime.now(), bid_size, 6)
#now to trim the serie every 60 second
if dt.datetime.now() > self.last_trim + self.moving_window_period:
self.__trim_data_series()
# Post-bootstrap - make trading decisions
# if self.strategy_params.is_bootstrap_completed():
# self.__recalculate_strategy_parameters_at_interval()
# self.__perform_trade_logic()
# self.__update_charts()
def __add_market_data(self, ticker_index, timestamp, value, col):
if col == 1:
self.prices.loc[timestamp, "price"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values ## Not sure how to handle this one
self.prices.sort_index(inplace=True)
elif col ==2:
self.prices.loc[timestamp, "size"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
elif col ==3:
self.prices.loc[timestamp, "ask_price"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
elif col ==4:
self.prices.loc[timestamp, "ask_size"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
elif col ==5:
self.prices.loc[timestamp, "bid_price"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
elif col ==6:
self.prices.loc[timestamp, "bid_size"] = float(value)
# self.prices = self.prices.fillna(method='ffill') # Clear NaN values
self.prices.sort_index(inplace=True)
def __stream_to_ohlc(self):
# stream = stream[["price","size"]]
# stream.is_copy = False
# stream.dropna(inplace=True, how='all')
try:
new_ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count"))
# very likely fuckery to be checked at the cutoff
t_stmp = self.prices.first_valid_index().replace(second=0, microsecond=0)
new_ohlc.loc[t_stmp, "open"] = float(self.prices['price'].dropna().head(1))
new_ohlc.loc[t_stmp, "close"] = float(self.prices['price'].dropna().tail(1))
new_ohlc.loc[t_stmp, "high"] = float(self.prices['price'].max())
new_ohlc.loc[t_stmp, "low"] = float(self.prices['price'].min())
new_ohlc.loc[t_stmp, "low"] = float(self.prices['price'].min())
new_ohlc.loc[t_stmp, "volume"] = float(self.prices['size'].sum())
new_ohlc.loc[t_stmp, "count"] = float(self.prices['size'].count())
return new_ohlc
except Exception, e:
print "f**k:", e
print self.prices
def __init__(self,
host='localhost',
port=4001,
client_id=130,
is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60),
test=False):
logging.basicConfig(format='%(asctime)s %(message)s')
self.test_logger = logging.getLogger('hftModelLogger')
self.test_logger.setLevel(logging.INFO)
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6)
self.timekeeper = None
self.parser = None
self.execution = None
self.strategy = None
self.handler = None
self.data_path = os.path.normpath(
os.path.join(os.path.curdir, "data.csv"))
self.ohlc_path = os.path.normpath(
os.path.join(os.path.curdir, "ohlc.csv"))
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
self.cur_mean = None
self.cur_sd = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
if not self.test:
self.handler = ExecutionHandler(self.conn)
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
else:
self.order_template = self.handler.create_contract(
settings.SYMBOL, settings.SECURITY, settings.EXCHANGE,
settings.EXPIRY, settings.CURRENCY)
self.signal = None
self.state = None
class HFTModel:
def __init__(self,
host='localhost',
port=4001,
client_id=130,
is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60),
test=False):
logging.basicConfig(format='%(asctime)s %(message)s')
self.test_logger = logging.getLogger('hftModelLogger')
self.test_logger.setLevel(logging.INFO)
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6)
self.timekeeper = None
self.parser = None
self.execution = None
self.strategy = None
self.handler = None
self.data_path = os.path.normpath(
os.path.join(os.path.curdir, "data.csv"))
self.ohlc_path = os.path.normpath(
os.path.join(os.path.curdir, "ohlc.csv"))
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
self.cur_mean = None
self.cur_sd = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
if not self.test:
self.handler = ExecutionHandler(self.conn)
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
else:
self.order_template = self.handler.create_contract(
settings.SYMBOL, settings.SECURITY, settings.EXCHANGE,
settings.EXPIRY, settings.CURRENCY)
self.signal = None
self.state = None
def _market_is_open(self):
tz_cme = pytz.timezone('America/Chicago')
cme_now = self.now.astimezone(tz_cme)
if cme_now.weekday() in [
0, 1, 2, 3
] and (cme_now >= cme_now.replace(hour=17, minute=0, second=0)
or cme_now < cme_now.replace(hour=16, minute=0, second=0)):
self.event_market_on.set()
elif cme_now.weekday() is 4 and cme_now < cme_now.replace(
hour=16, minute=0, second=0):
self.event_market_on.set()
elif cme_now.weekday() is 6 and cme_now >= cme_now.replace(
hour=17, minute=0, second=0):
self.event_market_on.set()
else:
self.event_market_on.clear()
# print "market is on:"
# print self.event_market_on.is_set()
def time_keeper(self):
while True:
print "executing and trading flag set?"
print self.handler.executing.is_set()
print self.handler.trading.is_set()
self.now = pytz.timezone('Singapore').localize(dt.datetime.now())
self._market_is_open()
# OHLC call
if self.last_trim is None:
self.last_trim = self.now
if self.now > self.last_trim + dt.timedelta(minutes=1):
if self.parser.running():
self.test_logger.error(
"parser thread is alive - timekeeper")
if self.strategy.running():
self.test_logger.error(
"strategy thread is alive - timekeeper")
if self.execution.running():
self.test_logger.error(
"execution thread is alive - timekeeper")
else:
self.test_logger.error(
"!!!execution thread is DEAD- timekeeper")
self.test_logger.error("timekeeper minute call")
try:
self.__request_historical_data(self.conn, initial=False)
except:
self.test_logger.error(
"req of update historicals failed - timekeep/hft")
try:
self.__run_indicators(self.ohlc)
except:
self.test_logger.error(
"runnning indicators failed - timekeep/hft")
self.conn.reqPositions()
self.update_norm_params()
if self.test:
print self.ohlc.tail()
time.sleep(
5
) #TODO horrible, horrible, but can't be bothered with a lock right now
# ML Call
if self.last_ml_call is None:
self.last_ml_call = self.now
if self.now > self.last_ml_call + dt.timedelta(minutes=5):
#logging.DEBUG("ML Call")
#logging.DEBUG(str(self.ohlc))
try:
self.flag = self.ml.call_ml(self.ohlc)
self.handler.flag = self.flag # this is stupid todo why why why
self.test_logger.error("I believe we will " +
self.handler.flag)
self.last_ml_call = self.now
except:
self.test_logger.error(
"!!! Call ML failed - timekeeper/hft")
time.sleep(1)
def __register_data_handlers(self, tick_event_handler,
universal_event_handler, order_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler, ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler, ib_message_type.tickPrice,
ib_message_type.tickSize)
self.conn.register(order_handler, ib_message_type.position,
ib_message_type.nextValidId,
ib_message_type.orderStatus,
ib_message_type.openOrder, ib_message_type.error)
def __init_stocks_data(self, symbols): #todo brutal hack-through
self.symbols = symbols
#here we'll store tick and size instead of multiple "symbols"
self.prices = pd.DataFrame(columns=(
"price", "size", "ask_price", "ask_size", "bid_price",
"bid_size")) #todomoved to execution # Init price storage
if not os.path.exists(self.data_path):
self.prices.to_csv(self.data_path)
self.ohlc = pd.DataFrame(columns=("open", "high", "low", "close",
"volume",
"count")) # Init ohlc storage
if not os.path.exists(self.ohlc_path):
self.ohlc.to_csv(self.ohlc_path)
print "checked for csv file"
#Now I have only one "symbol" TODO: clean that stuff
stock_symbol = self.symbols
contract = self.ib_util.create_stock_contract(stock_symbol)
#self.stocks_data[stock_symbol] = StockData(contract)
#this is redundant but required to scaffold/test
def create_contract(self, symbol, sec_type, exch, expiry, curr):
"""Create a Contract object defining what will
be purchased, at which exchange and in which currency.
symbol - The ticker symbol for the contract
sec_type - The security type for the contract ('FUT' = Future)
exch - The exchange to carry out the contract on
prim_exch - The primary exchange to carry out the contract on
curr - The currency in which to purchase the contract"""
contract = Contract()
contract.m_symbol = symbol
contract.m_secType = sec_type
contract.m_exchange = exch
contract.m_expiry = expiry
contract.m_currency = curr
return contract
def __request_streaming_data(self, ib_conn):
# Stream market data.
ib_conn.reqMktData(1, self.order_template, datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
# time.sleep(5)
def __request_historical_data(self, ib_conn, initial=True):
""" the same method can be used for scheduled calls"""
# self.ohlc_ok.acquire()
if initial:
duration = datatype.DURATION_2_HR
else:
duration = datatype.DURATION_1_MIN
ib_conn.reqHistoricalData(1, self.order_template,
time.strftime(datatype.DATE_TIME_FORMAT),
duration, datatype.BAR_SIZE_1_MIN,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL, datatype.DATEFORMAT_STRING)
time.sleep(1)
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
pass
#self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
pass
else:
print msg
def null_handler(self, msg):
pass
def __on_historical_data(self, msg):
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed()
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self):
#self.lock.release()
self.last_trim = self.ohlc.index[-1] + self.moving_window_period
print "trim time properly set now %s" % self.last_trim
print "start position is:" + str(self.handler.position)
self.__run_indicators(self.ohlc)
self.ohlc.to_csv(self.ohlc_path)
def __add_historical_data(self, ticker_index, msg):
if self.test:
print "adding histo line"
timestamp = pytz.timezone('Singapore').localize(
dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT))
self.__add_ohlc_data(timestamp, msg.open, msg.high, msg.low, msg.close,
msg.volume, msg.count)
def __add_ohlc_data(self, timestamp, op, hi, lo, close, vol, cnt):
self.ohlc.loc[timestamp, "open"] = float(op)
self.ohlc.loc[timestamp, "high"] = float(hi)
self.ohlc.loc[timestamp, "low"] = float(lo)
self.ohlc.loc[timestamp, "close"] = float(close)
self.ohlc.loc[timestamp, "volume"] = float(vol)
self.ohlc.loc[timestamp, "count"] = float(cnt)
#
def __run_indicators(self, ohlc):
#hardcoding ML munging parameters now
ohlc['returns'] = ta.ROC(np.asarray(ohlc['close']).astype(float))
ohlc['sma'] = ta.SMA(np.asarray(ohlc['close']).astype(float), 10)
ohlc['lma'] = ta.SMA(np.asarray(ohlc['close']).astype(float), 120)
ohlc['rsi'] = ta.RSI(np.asarray(ohlc['close']).astype(float))
ohlc['atr'] = ta.ATR(
np.asarray(ohlc['high']).astype(float),
np.asarray(ohlc['low']).astype(float),
np.asarray(ohlc['close']).astype(float), 10)
ohlc['monday'] = np.where(ohlc.index.weekday == 0, 1, 0)
#bellow is because I don't know how to np.where with multiple conditions
ohlc['roll'] = np.where(ohlc.index.month % 3 == 0, 1, 0)
#hackish as balls:
# ohlc.index = ohlc.index.tz_localize("Singapore")
ohlc["busy"] = np.where(
ohlc.index.tz_convert("America/Chicago").hour >= 9,
np.where(
ohlc.index.tz_convert("America/Chicago").hour <= 14, 1, 0), 0)
def update_norm_params(self):
try:
prices = self.handler.prices["price"]
except:
self.test_logger.error(
"getting price from handler failed - update norm/hft")
sgp_tz = pytz.timezone('Singapore')
try:
prices.index = prices.index.tz_localize(sgp_tz)
#prices.to_pick (os.path.join(os.path.curdir,"prices_f_norm.csv"))
except:
self.test_logger.error(
"prices localization failed - update norm/hft")
#
try:
prices = prices.dropna()
except:
self.test_logger.error("dropna failed - update norm/hft")
try:
if prices.index.max() - prices.index.min() > dt.timedelta(
seconds=60) and self.last_trim is not None:
prices = prices[
prices.index > prices.index.max() - dt.timedelta(
seconds=60)] #todo no trim of prices as of now
print "trimmed prices"
print prices
#prices.to_pickle(os.path.join(os.path.curdir(), "prices.pkl"))
except:
self.test_logger.error(
"minute trim of prices failed - update norm/hft")
if len(prices) != 0:
last_price = prices.iloc[-1]
try:
my_cur_mean = round(np.mean(prices), 2)
self.handler.cur_mean = my_cur_mean
except:
self.test_logger.error("mean update failed- update norm/hft")
try:
prices = prices.diff()
prices = prices.dropna()
prices = prices**2
except:
self.test_logger.error("diff/square failed- update norm/hft")
tdiffs = list()
try:
for i in range(1, len(prices)):
tdiffs.append((prices.index[i] -
prices.index[i - 1]).total_seconds())
prices = prices.ix[1:]
print prices
except:
self.test_logger.error(
"time diffs creation failed - update norm/hft")
try:
my_cur_sd = round(sqrt(sum(prices * tdiffs) / len(prices)), 2)
self.handler.cur_sd = my_cur_sd
except:
self.test_logger.error("StDev udpate failed - update norm/hft")
self.handler.cur_sd = 2 * settings.STOP_OFFSET # in case the stdev failed
self.test_logger.error(
"update norm parameters completed - update norm/hft")
self.last_trim = self.now
def __cancel_market_data_request(self):
self.conn.cancelMktData(1)
time.sleep(1)
def spawn(self):
print "execution thread spawned"
self.handler = ExecutionHandler(self.conn)
def start(self, symbols):
self.test_logger.info("Started Requests !!!!")
self.conn.connect() # Get IB connection object
self.__init_stocks_data(symbols)
self.test_logger.info("Init Stock")
self.test_logger.info("Request Market Data")
self.__request_streaming_data(self.conn)
self.test_logger.info("Request Position")
self.conn.reqPositions()
self.test_logger.info("Request Historicals")
self.__request_historical_data(self.conn)
time.sleep(1)
if self.handler.position != 0:
self.test_logger.info("Squaring off for a clean start")
self.handler.neutralize()
try:
#self.time_keeper()
time.sleep(5)
self.test_logger.info("I hope Ihave ohlc now, from hft")
print self.ohlc.tail(5)
self.test_logger.info("call ML first time - HFT")
self.flag = self.ml.call_ml(self.ohlc)
self.handler.flag = self.flag # this is stupid
time.sleep(3)
print "I believe we will " + self.flag
self.test_logger.info("Spawn concurrent processes")
self.timekeeper = self.executor.submit(self.time_keeper)
self.test_logger.info("Time keeper spawned")
time.sleep(1)
self.parser = self.executor.submit(self.handler.queue_parser)
self.test_logger.info("Parser spawned")
time.sleep(5)
self.update_norm_params()
self.test_logger.info("First normalized parameters passed")
time.sleep(5)
self.test_logger.info("Spawning strategy handler")
self.strategy = self.executor.submit(self.handler.trading_loop)
self.test_logger.info("Spawning execution handler")
self.execution = self.executor.submit(self.handler.order_execution)
except (Exception, KeyboardInterrupt):
print "Exception:"
print "Cancelling...",
self.__cancel_market_data_request()
print "killing all orders"
self.handler.kill_em_all()
# self.monitor.close_stream()
print "Disconnecting..."
time.sleep(5)
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
def stop(self):
os.remove(os.path.normpath(os.path.join(os.path.curdir, "data.csv")))
os.remove(os.path.normpath(os.path.join(os.path.curdir, "ohlc.csv")))
self.__cancel_market_data_request()
#self.monitor.close_stream()
print "Disconnecting..."
self.conn.disconnect()
# self.store.close()
def rekindle_execution(self):
self.execution = self.executor.submit(self.handler.trading_loop)
class ATS:
def __init__(self, host='localhost', port=4001,
client_id=101, is_use_gateway=False, evaluation_time_secs=20,
resample_interval_secs='30s',
moving_window_period=dt.timedelta(hours=1)):
self.moving_window_period = moving_window_period
self.chart = Chart()
self.ib_util = IBUtil()
self.strategy_params = StrategyParameters(evaluation_time_secs,
resample_interval_secs)
self.stocks_data = {}
self.symbols = None
self.account_code = ""
self.prices = None
self.trade_qty = 0
self.order_id = 0
self.lock = threading.Lock()
self.conn = ibConnection() if is_use_gateway else \
Connection.create(host=host, port=port, clientId=client_id)
self.__register_data_handlers(self.__on_tick_event,
self.__event_handler)
def __perform_trade_logic(self):
volatility_ratio = self.strategy_params.get_volatility_ratio()
is_up_trend, is_down_trend = volatility_ratio > 1, volatility_ratio < 1
is_overbought, is_oversold = self.__is_overbought_or_oversold()
is_buy_signal, is_sell_signal = (is_up_trend and is_oversold), \
(is_down_trend and is_overbought)
symbol_a = self.symbols[0]
position = self.stocks_data[symbol_a].position
is_position_closed, is_short, is_long = \
(position == 0), (position < 0), (position > 0)
upnl, rpnl = self.__calculate_pnls()
signal_text = \
"BUY" if is_buy_signal else "SELL" if is_sell_signal else "NONE"
console_output = '\r[%s] signal=%s, position=%s UPnL=%s RPnL=%s\r' % \
(dt.datetime.now(), signal_text, position, upnl, rpnl)
sys.stdout.write(console_output)
sys.stdout.flush()
if is_position_closed and is_sell_signal:
print "=================================="
print "OPEN SHORT POSIITON: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
elif is_position_closed and is_buy_signal:
print "=================================="
print "OPEN LONG POSIITON: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_short and is_buy_signal:
print "=================================="
print "CLOSE SHORT POSITION: BUY A SELL B"
print "=================================="
self.__place_spread_order(self.trade_qty)
elif is_long and is_sell_signal:
print "=================================="
print "CLOSE LONG POSITION: SELL A BUY B"
print "=================================="
self.__place_spread_order(-self.trade_qty)
def __recalculate_strategy_parameters_at_interval(self):
if self.strategy_params.is_evaluation_time_elapsed():
self.__calculate_strategy_params()
self.strategy_params.set_new_evaluation_time()
print '[%s] === Beta re-evaluated ===' % dt.datetime.now()
def __calculate_strategy_params(self):
[symbol_a, symbol_b] = self.symbols
filled_prices = self.prices.fillna(method='ffill')
resampled = filled_prices.resample(
self.strategy_params.resample_interval_secs, fill_method='ffill')\
.dropna()
mean = resampled.mean()
beta = mean[symbol_a] / mean[symbol_b]
stddevs = resampled.pct_change().dropna().std()
volatility_ratio = stddevs[symbol_a] / stddevs[symbol_b]
self.strategy_params.add_indicators(beta, volatility_ratio)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
def __init_stocks_data(self, symbols):
self.symbols = symbols
self.prices = pd.DataFrame(columns=symbols)
for stock_symbol in symbols:
contract = self.ib_util.create_stock_contract(stock_symbol)
self.stocks_data[stock_symbol] = StockData(contract)
def __request_streaming_data(self, ib_conn):
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
ib_conn.reqMktData(index,
stock_data.contract,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
time.sleep(1)
ib_conn.reqAccountUpdates(True, self.account_code)
def __request_historical_data(self, ib_conn):
self.lock.acquire()
try:
for index, (key, stock_data) in enumerate(
self.stocks_data.iteritems()):
stock_data.is_storing_data = True
ib_conn.reqHistoricalData(
index,
stock_data.contract,
time.strftime(datatype.DATE_TIME_FORMAT),
datatype.DURATION_1_HR,
datatype.BAR_SIZE_5_SEC,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
finally:
self.lock.release()
def __wait_for_download_completion(self):
is_waiting = True
while is_waiting:
is_waiting = False
self.lock.acquire()
try:
for symbol in self.stocks_data.keys():
if self.stocks_data[symbol].is_storing_data:
is_waiting = True
finally:
self.lock.release()
if is_waiting:
time.sleep(1)
def __place_spread_order(self, qty):
[symbol_a, symbol_b] = self.symbols
self.__send_order(symbol_a, qty)
self.__send_order(symbol_b, -qty)
def __send_order(self, symbol, qty):
stock_data = self.stocks_data[symbol]
order = self.ib_util.create_stock_order(abs(qty), qty > 0)
self.conn.placeOrder(self.__generate_order_id(),
stock_data.contract,
order)
stock_data.add_to_position(qty)
def __generate_order_id(self):
next_order_id = self.order_id
self.order_id += 1
return next_order_id
def __is_overbought_or_oversold(self):
[symbol_a, symbol_b] = self.symbols
leg_a_last_price = self.prices[symbol_a].values[-1]
leg_b_last_price = self.prices[symbol_b].values[-1]
expected_leg_a_price = \
leg_b_last_price * self.strategy_params.get_beta()
is_overbought = \
leg_a_last_price < expected_leg_a_price
is_oversold = \
leg_a_last_price > expected_leg_a_price
return is_overbought, is_oversold
def __on_portfolio_update(self, msg):
for key, stock_data in self.stocks_data.iteritems():
if stock_data.contract.m_symbol == msg.contract.m_symbol:
stock_data.update_position(msg.position,
msg.marketPrice,
msg.marketValue,
msg.averageCost,
msg.unrealizedPNL,
msg.realizedPNL,
msg.accountName)
return
def __calculate_pnls(self):
upnl, rpnl = 0, 0
for key, stock_data in self.stocks_data.iteritems():
upnl += stock_data.unrealized_pnl
rpnl += stock_data.realized_pnl
return upnl, rpnl
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
self.account_code = msg.accountsList
elif msg.typeName == datatype.MSG_TYPE_NEXT_ORDER_ID:
self.order_id = msg.orderId
else:
print msg
def __on_historical_data(self, msg):
print msg
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed(ticker_index)
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self, ticker_index):
self.lock.acquire()
try:
symbol = self.symbols[ticker_index]
self.stocks_data[symbol].is_storing_data = False
finally:
self.lock.release()
def __add_historical_data(self, ticker_index, msg):
timestamp = dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT)
self.__add_market_data(ticker_index, timestamp, msg.close)
def __on_tick_event(self, msg):
ticker_id = msg.tickerId
field_type = msg.field
if field_type == datatype.FIELD_LAST_PRICE:
last_price = msg.price
self.__add_market_data(ticker_id, dt.datetime.now(), last_price)
self.__trim_data_series()
if self.strategy_params.is_bootstrap_completed():
self.__recalculate_strategy_parameters_at_interval()
self.__perform_trade_logic()
self.__update_charts()
def __add_market_data(self, ticker_index, timestamp, price):
symbol = self.symbols[ticker_index]
self.prices.loc[timestamp, symbol] = float(price)
self.prices = self.prices.fillna(method='ffill')
self.prices.sort_index(inplace=True)
def __update_charts(self):
if len(self.prices) > 0 and len(self.strategy_params.indicators) > 0:
self.chart.display_chart(self.prices,
self.strategy_params.indicators)
def __trim_data_series(self):
cutoff_timestamp = dt.datetime.now() - self.moving_window_period
self.prices = self.prices[self.prices.index >= cutoff_timestamp]
self.strategy_params.trim_indicators_series(cutoff_timestamp)
@staticmethod
def __print_elapsed_time(start_time):
elapsed_time = time.time() - start_time
print "Completed in %.3f seconds." % elapsed_time
def __cancel_market_data_request(self):
for i, symbol in enumerate(self.symbols):
self.conn.cancelMktData(i)
time.sleep(1)
def start(self, symbols, trade_qty):
print "ATS model started."
self.trade_qty = trade_qty
self.conn.connect()
self.__init_stocks_data(symbols)
self.__request_streaming_data(self.conn)
print "Bootstrapping the model..."
start_time = time.time()
self.__request_historical_data(self.conn)
self.__wait_for_download_completion()
self.strategy_params.set_bootstrap_completed()
self.__print_elapsed_time(start_time)
print "Calculating strategy parameters..."
start_time = time.time()
self.__calculate_strategy_params()
self.__print_elapsed_time(start_time)
print "Trading started."
try:
self.__update_charts()
while True:
time.sleep(1)
except Exception, e:
print "Exception:", e
print "Cancelling...",
self.__cancel_market_data_request()
print "Disconnecting..."
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60), test=False):
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.stocks_data = {} # Dictionary storing StockData objects.REFACTOR
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
#self.lock = Lock()
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
self.timekeeper = None
self.parser = None
self.execution = None
self.handler = None
self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv"))
self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))
self.last_trim = None
#range/trend flag
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
# self.last_trade = None
# self.last_bid = None
# self.last_ask = None
# self.cur_mean = None
# self.cur_sd = None
# self.cur_zscore = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
#self.thread = threading.Thread(target=self.spawn())
#self.thread.start()
if not self.test:
self.handler = ExecutionHandler(self.conn)
#
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract("CL", "FUT", "NYMEX", "201606", "USD")
else:
self.order_template = self.handler.create_contract("CL", "FUT", "NYMEX", "201606", "USD")#todo duplicate with execution handler
self.signal = None
self.state = None
class HFTModel:
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60), test=False):
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.stocks_data = {} # Dictionary storing StockData objects.REFACTOR
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
#self.lock = Lock()
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=10)
self.timekeeper = None
self.parser = None
self.execution = None
self.handler = None
self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv"))
self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))
self.last_trim = None
#range/trend flag
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
# self.last_trade = None
# self.last_bid = None
# self.last_ask = None
# self.cur_mean = None
# self.cur_sd = None
# self.cur_zscore = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
#self.thread = threading.Thread(target=self.spawn())
#self.thread.start()
if not self.test:
self.handler = ExecutionHandler(self.conn)
#
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract("CL", "FUT", "NYMEX", "201606", "USD")
else:
self.order_template = self.handler.create_contract("CL", "FUT", "NYMEX", "201606", "USD")#todo duplicate with execution handler
self.signal = None
self.state = None
def _market_is_open(self):
tz_cme = pytz.timezone('America/Chicago')
cme_now = self.now.astimezone(tz_cme)
if cme_now.weekday() in [0, 1, 2, 3] and (
cme_now >= cme_now.replace(hour=17, minute=0, second=0) or cme_now < cme_now.replace(hour=16,
minute=0,
second=0)):
self.event_market_on.set()
elif cme_now.weekday() is 4 and cme_now < cme_now.replace(hour=16, minute=0, second=0):
self.event_market_on.set()
elif cme_now.weekday() is 6 and cme_now >= cme_now.replace(hour=17, minute=0, second=0):
self.event_market_on.set()
else:
self.event_market_on.clear()
# print "market is on:"
# print self.event_market_on.is_set()
def time_keeper(self):
#self.traffic_light.wait()
while True:
# if self.test == True:
print "timekeeper alive"
self.now = pytz.timezone('Singapore').localize(dt.datetime.now())
self._market_is_open()
# OHLC call
if self.last_trim is None:
self.last_trim = self.now
if self.now > self.last_trim + dt.timedelta(minutes=1):
print "call trim from scheduler"
self.__request_historical_data(self.conn,initial=False)
self.__run_indicators(self.ohlc)
print "execution process is alive:"
print self.execution.running()
print "parser process is alive:"
print self.parser.running()
print "position is:"
print self.handler.position
print "I have curmean, cursd, last_trade in execution"
print str(self.handler.cur_mean)
print str(self.handler.cur_sd)
print str(self.handler.last_trade)
print "request position:"
self.conn.reqPositions()
self.update_norm_params()
self.last_trim = self.now
if self.test:
print self.ohlc.tail()
time.sleep(5)#TODO horrible, horrible, but can't be bothered with a lock right now
# ML Call
if self.last_ml_call is None:
self.last_ml_call = self.now
if self.now > self.last_ml_call + dt.timedelta(minutes=5):
print "ML Call"
logging.DEBUG("ML Call")
logging.DEBUG(str(self.ohlc))
try:
self.ml.call_ml(self.ohlc)
except:
print "ml call failed"
print "ML call complete"
self.last_ml_call = self.now
time.sleep(1)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler,order_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
self.conn.register(order_handler,
ib_message_type.position,
ib_message_type.nextValidId,
ib_message_type.orderStatus,
ib_message_type.openOrder,
ib_message_type.error)
def __init_stocks_data(self, symbols):#todo brutal hack-through
self.symbols = symbols
#here we'll store tick and size instead of multiple "symbols"
# self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size"))#todomoved to execution # Init price storage
if not os.path.exists(self.data_path):
self.prices.to_csv(self.data_path)
self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage
if not os.path.exists(self.ohlc_path):
self.ohlc.to_csv(self.ohlc_path)
print "checked for csv file"
#Now I have only one "symbol" TODO: clean that stuff
stock_symbol = self.symbols
contract = self.ib_util.create_stock_contract(stock_symbol)
self.stocks_data[stock_symbol] = StockData(contract)
#this is redundant but required to scaffold/test
def create_contract(self, symbol, sec_type, exch, expiry, curr):
"""Create a Contract object defining what will
be purchased, at which exchange and in which currency.
symbol - The ticker symbol for the contract
sec_type - The security type for the contract ('FUT' = Future)
exch - The exchange to carry out the contract on
prim_exch - The primary exchange to carry out the contract on
curr - The currency in which to purchase the contract"""
contract = Contract()
contract.m_symbol = symbol
contract.m_secType = sec_type
contract.m_exchange = exch
contract.m_expiry = expiry
contract.m_currency = curr
return contract
def __request_streaming_data(self, ib_conn):
# Stream market data.
ib_conn.reqMktData(1,
self.order_template,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
# time.sleep(5)
# Stream account updates DEACTIVATED FOR NOW
#ib_conn.reqAccountUpdates(True, self.account_code)
def __request_historical_data(self, ib_conn, initial=True):
""" the same method can be used for scheduled calls"""
# self.ohlc_ok.acquire()
if initial:
duration = datatype.DURATION_2_HR
else:
duration = datatype.DURATION_1_MIN
ib_conn.reqHistoricalData(
1,
self.order_template,
time.strftime(datatype.DATE_TIME_FORMAT),
duration,
datatype.BAR_SIZE_1_MIN,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
def __on_portfolio_update(self, msg):
for key, stock_data in self.stocks_data.iteritems():
if stock_data.contract.m_symbol == msg.contract.m_symbol:
if dt.datetime.now(self.tz) > self.last_trim + self.moving_window_period:
stock_data.update_position(msg.position,
msg.marketPrice,
msg.marketValue)
# ,
# msg.averageCost,
# msg.unrealizedPNL,
# msg.realizedPNL,
# msg.accountName)
return
# def __calculate_pnls(self):
# upnl, rpnl = 0, 0
# for key, stock_data in self.stocks_data.iteritems():
# upnl += stock_data.unrealized_pnl
# rpnl += stock_data.realized_pnl
# return upnl, rpnl
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
pass
else:
print msg
def null_handler(self,msg):
pass
def __on_historical_data(self, msg):
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed()
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self):
#self.lock.release()
self.last_trim = self.ohlc.index[-1]+self.moving_window_period
print "trim time properly set now %s" % self.last_trim
print "start position is:" + str(self.handler.position)
self.__run_indicators(self.ohlc)
self.ohlc.to_csv(self.ohlc_path)
# self.ohlc.to_pickle("/Users/maxime_back/Documents/avocado/ohlc.pickle")
def __add_historical_data(self, ticker_index, msg):
if self.test:
print "adding histo line"
timestamp = pytz.timezone('Singapore').localize(dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT))
self.__add_ohlc_data(timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count)
def __add_ohlc_data(self, timestamp, op, hi ,lo,close,vol,cnt):
self.ohlc.loc[timestamp, "open"] = float(op)
self.ohlc.loc[timestamp, "high"] = float(hi)
self.ohlc.loc[timestamp, "low"] = float(lo)
self.ohlc.loc[timestamp, "close"] = float(close)
self.ohlc.loc[timestamp, "volume"] = float(vol)
self.ohlc.loc[timestamp, "count"] = float(cnt)
#
def __run_indicators(self, ohlc):
#hardcoding ML munging parameters now
ohlc['returns']=ta.ROC(np.asarray(ohlc['close']).astype(float))
ohlc['sma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 10)
ohlc['lma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 120)
ohlc['rsi']=ta.RSI(np.asarray(ohlc['close']).astype(float))
ohlc['atr']=ta.ATR(np.asarray(ohlc['high']).astype(float),np.asarray(ohlc['low']).astype(float),np.asarray(ohlc['close']).astype(float),10)
ohlc['monday'] = np.where(ohlc.index.weekday == 0,1,0)
#bellow is because I don't know how to np.where with multiple conditions
ohlc['roll'] = np.where(ohlc.index.month % 3 == 0,1,0)
#hackish as balls:
# ohlc.index = ohlc.index.tz_localize("Singapore")
ohlc["busy"] = np.where(ohlc.index.tz_convert("America/Chicago").hour >= 9,np.where(ohlc.index.tz_convert("America/Chicago").hour <= 14,1,0),0)
def update_norm_params(self):
# print " updating feeder params for zscore"
print "update norm got prices from handler, len:"
print len(self.handler.prices)
try:
prices = self.handler.prices["price"]
sgp_tz = pytz.timezone('Singapore')
prices.index = prices.index.tz_localize(sgp_tz)
prices.to_csv(os.path.join(os.path.curdir,"prices_f_norm.csv"))
# print " got prices"
prices = prices.dropna()
print "sd crapping potential ahead"
print "last trim in update norm"
print self.last_trim
if prices.index.max()-prices.index.min() > dt.timedelta(seconds=60) and self.last_trim is not None:
prices = prices[prices.index > self.last_trim]#todo no trim of prices
# prices = prices.tail(15)#ik,ik
print "sd crap avoided"
except:
print "update norm FAILED - from update norm"
try:
if len(prices) !=0:
last_price = prices.iloc[-1]
self.handler.cur_mean = round(np.mean(prices),2)
#logging.debug("updated mean")
#print last_price
prices = prices.diff()
prices = prices.dropna()
prices = prices**2
tdiffs = list()
for i in range(1,len(prices)):
tdiffs.append((prices.index[i]-prices.index[i-1]).total_seconds())
prices = prices.ix[1:]
self.handler.cur_sd = round(sqrt(sum(prices * tdiffs)/len(prices)), 2)
logging.debug("updated sd")
logging.debug(str(self.handler.cur_sd))
# self.cur_zscore = (last_price - self.cur_mean)/self.cur_sd
#print(self.cur_zscore)
except:
print "update normfailed - second halh - from update norm"
def __cancel_market_data_request(self):
self.conn.cancelMktData(1)
time.sleep(1)
#recycling zscore spawn to thread the handler
def spawn(self):
print "execution thread spawned"
self.handler = ExecutionHandler(self.conn)
def start(self, symbols):
print "Start sequence"
logging.debug("started requests")
self.conn.connect() # Get IB connection object
self.__init_stocks_data(symbols)
print "init stock"
print "request mkt data"
self.__request_streaming_data(self.conn)
print "request position"
self.conn.reqPositions()
print "request historicals"
self.__request_historical_data(self.conn)
time.sleep(1)
if self.handler.position !=0:
print "squaring position for a clean start"
self.handler.neutralize()
try:
print "getting ohlc data now"
#self.time_keeper()
time.sleep(5)
print "pray I have them now"
print self.ohlc.tail(5)
print "calling ML for the first time"
self.flag = self.ml.call_ml(self.ohlc)
self.handler.flag = self.flag# this is stupid
time.sleep(3)
print "I believe we will "+ self.flag
# if self.test:
# print self.ohlc
# time.sleep(60)
# print "now calling for update"
# self.__request_historical_data(self.conn,initial=False)
# print self.ohlc
print "spawn concurrent processes"
self.timekeeper = self.executor.submit(self.time_keeper)
print "timekeeper spawned"
time.sleep(1)
self.parser = self.executor.submit(self.handler.queue_parser)
print "parser spawned"
time.sleep(5)
self.update_norm_params()
print "sd/mean: passed"
time.sleep(5)
print "handler spawned"
self.execution = self.executor.submit(self.handler.trading_loop)
except (Exception, KeyboardInterrupt):
print "Exception:"
print "Cancelling...",
self.__cancel_market_data_request()
print "killing all orders"
self.handler.kill_em_all()
# self.monitor.close_stream()
print "Disconnecting..."
time.sleep(5)
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
def stop(self):
os.remove(os.path.normpath(os.path.join(os.path.curdir,"data.csv")))
os.remove(os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")))
self.__cancel_market_data_request()
#self.monitor.close_stream()
print "Disconnecting..."
self.conn.disconnect()
# self.store.close()
def spawn_test(self):
self.traffic_light.wait()
print "f**k spawns"
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60), test=False):
logging.basicConfig(format='%(asctime)s %(message)s')
self.test_logger = logging.getLogger('hftModelLogger')
self.test_logger.setLevel(logging.INFO)
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6)
self.timekeeper = None
self.parser = None
self.execution = None
self.strategy = None
self.handler = None
self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv"))
self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
self.cur_mean = None
self.cur_sd = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
if not self.test:
self.handler = ExecutionHandler(self.conn)
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
else:
self.order_template = self.handler.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
self.signal = None
self.state = None
class HFTModel:
def __init__(self, host='localhost', port=4001,
client_id=130, is_use_gateway=False,
moving_window_period=dt.timedelta(seconds=60), test=False):
logging.basicConfig(format='%(asctime)s %(message)s')
self.test_logger = logging.getLogger('hftModelLogger')
self.test_logger.setLevel(logging.INFO)
self.test = test
self.tz = pytz.timezone('Singapore')
self.moving_window_period = moving_window_period
self.ib_util = IBUtil()
self.symbols = None # List of current symbols
self.account_code = ""
self.prices = None # Store last prices in a DataFrame
self.ohlc = None # I need another store for minute data (I think)
self.buffer = list()
self.trade_qty = 0
self.traffic_light = Event()
self.ohlc_ok = Lock()
self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=6)
self.timekeeper = None
self.parser = None
self.execution = None
self.strategy = None
self.handler = None
self.data_path = os.path.normpath(os.path.join(os.path.curdir,"data.csv"))
self.ohlc_path = os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv"))
self.flag = None
self.event_market_on = Event()
self.ml = MLcall()
self.last_trim = None
self.last_ml_call = None
self.cur_mean = None
self.cur_sd = None
# Use ibConnection() for TWS, or create connection for API Gateway
self.conn = Connection.create(host=host, port=port, clientId=client_id)
if not self.test:
self.handler = ExecutionHandler(self.conn)
#third handler should register properly si Dieu veut
if self.test:
self.__register_data_handlers(self.null_handler,
self.__event_handler,
self.null_handler)
else:
self.__register_data_handlers(self.handler.on_tick_event,
self.__event_handler,
self.handler._reply_handler)
if self.test:
self.order_template = self.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
else:
self.order_template = self.handler.create_contract(settings.SYMBOL,
settings.SECURITY,
settings.EXCHANGE,
settings.EXPIRY,
settings.CURRENCY)
self.signal = None
self.state = None
def _market_is_open(self):
tz_cme = pytz.timezone('America/Chicago')
cme_now = self.now.astimezone(tz_cme)
if cme_now.weekday() in [0, 1, 2, 3] and (
cme_now >= cme_now.replace(hour=17, minute=0, second=0) or cme_now < cme_now.replace(hour=16,
minute=0,
second=0)):
self.event_market_on.set()
elif cme_now.weekday() is 4 and cme_now < cme_now.replace(hour=16, minute=0, second=0):
self.event_market_on.set()
elif cme_now.weekday() is 6 and cme_now >= cme_now.replace(hour=17, minute=0, second=0):
self.event_market_on.set()
else:
self.event_market_on.clear()
# print "market is on:"
# print self.event_market_on.is_set()
def time_keeper(self):
while True:
print "executing and trading flag set?"
print self.handler.executing.is_set()
print self.handler.trading.is_set()
self.now = pytz.timezone('Singapore').localize(dt.datetime.now())
self._market_is_open()
# OHLC call
if self.last_trim is None:
self.last_trim = self.now
if self.now > self.last_trim + dt.timedelta(minutes=1):
if self.parser.running():
self.test_logger.error("parser thread is alive - timekeeper")
if self.strategy.running():
self.test_logger.error("strategy thread is alive - timekeeper")
if self.execution.running():
self.test_logger.error("execution thread is alive - timekeeper")
else:
self.test_logger.error("!!!execution thread is DEAD- timekeeper")
self.test_logger.error("timekeeper minute call")
try:
self.__request_historical_data(self.conn,initial=False)
except:
self.test_logger.error("req of update historicals failed - timekeep/hft")
try:
self.__run_indicators(self.ohlc)
except:
self.test_logger.error("runnning indicators failed - timekeep/hft")
self.conn.reqPositions()
self.update_norm_params()
if self.test:
print self.ohlc.tail()
time.sleep(5)#TODO horrible, horrible, but can't be bothered with a lock right now
# ML Call
if self.last_ml_call is None:
self.last_ml_call = self.now
if self.now > self.last_ml_call + dt.timedelta(minutes=5):
#logging.DEBUG("ML Call")
#logging.DEBUG(str(self.ohlc))
try:
self.flag = self.ml.call_ml(self.ohlc)
self.handler.flag = self.flag # this is stupid todo why why why
self.test_logger.error("I believe we will " + self.handler.flag)
self.last_ml_call = self.now
except:
self.test_logger.error("!!! Call ML failed - timekeeper/hft")
time.sleep(1)
def __register_data_handlers(self,
tick_event_handler,
universal_event_handler,order_handler):
self.conn.registerAll(universal_event_handler)
self.conn.unregister(universal_event_handler,
ib_message_type.tickSize,
ib_message_type.tickPrice,
ib_message_type.tickString,
ib_message_type.tickGeneric,
ib_message_type.tickOptionComputation)
self.conn.register(tick_event_handler,
ib_message_type.tickPrice,
ib_message_type.tickSize)
self.conn.register(order_handler,
ib_message_type.position,
ib_message_type.nextValidId,
ib_message_type.orderStatus,
ib_message_type.openOrder,
ib_message_type.error)
def __init_stocks_data(self, symbols):#todo brutal hack-through
self.symbols = symbols
#here we'll store tick and size instead of multiple "symbols"
self.prices = pd.DataFrame(columns=("price","size","ask_price","ask_size","bid_price","bid_size"))#todomoved to execution # Init price storage
if not os.path.exists(self.data_path):
self.prices.to_csv(self.data_path)
self.ohlc = pd.DataFrame(columns=("open","high","low","close","volume","count")) # Init ohlc storage
if not os.path.exists(self.ohlc_path):
self.ohlc.to_csv(self.ohlc_path)
print "checked for csv file"
#Now I have only one "symbol" TODO: clean that stuff
stock_symbol = self.symbols
contract = self.ib_util.create_stock_contract(stock_symbol)
#self.stocks_data[stock_symbol] = StockData(contract)
#this is redundant but required to scaffold/test
def create_contract(self, symbol, sec_type, exch, expiry, curr):
"""Create a Contract object defining what will
be purchased, at which exchange and in which currency.
symbol - The ticker symbol for the contract
sec_type - The security type for the contract ('FUT' = Future)
exch - The exchange to carry out the contract on
prim_exch - The primary exchange to carry out the contract on
curr - The currency in which to purchase the contract"""
contract = Contract()
contract.m_symbol = symbol
contract.m_secType = sec_type
contract.m_exchange = exch
contract.m_expiry = expiry
contract.m_currency = curr
return contract
def __request_streaming_data(self, ib_conn):
# Stream market data.
ib_conn.reqMktData(1,
self.order_template,
datatype.GENERIC_TICKS_NONE,
datatype.SNAPSHOT_NONE)
# time.sleep(5)
def __request_historical_data(self, ib_conn, initial=True):
""" the same method can be used for scheduled calls"""
# self.ohlc_ok.acquire()
if initial:
duration = datatype.DURATION_2_HR
else:
duration = datatype.DURATION_1_MIN
ib_conn.reqHistoricalData(
1,
self.order_template,
time.strftime(datatype.DATE_TIME_FORMAT),
duration,
datatype.BAR_SIZE_1_MIN,
datatype.WHAT_TO_SHOW_TRADES,
datatype.RTH_ALL,
datatype.DATEFORMAT_STRING)
time.sleep(1)
def __event_handler(self, msg):
if msg.typeName == datatype.MSG_TYPE_HISTORICAL_DATA:
self.__on_historical_data(msg)
elif msg.typeName == datatype.MSG_TYPE_UPDATE_PORTFOLIO:
pass
#self.__on_portfolio_update(msg)
elif msg.typeName == datatype.MSG_TYPE_MANAGED_ACCOUNTS:
pass
else:
print msg
def null_handler(self,msg):
pass
def __on_historical_data(self, msg):
ticker_index = msg.reqId
if msg.WAP == -1:
self.__on_historical_data_completed()
else:
self.__add_historical_data(ticker_index, msg)
def __on_historical_data_completed(self):
#self.lock.release()
self.last_trim = self.ohlc.index[-1]+self.moving_window_period
print "trim time properly set now %s" % self.last_trim
print "start position is:" + str(self.handler.position)
self.__run_indicators(self.ohlc)
self.ohlc.to_csv(self.ohlc_path)
def __add_historical_data(self, ticker_index, msg):
if self.test:
print "adding histo line"
timestamp = pytz.timezone('Singapore').localize(dt.datetime.strptime(msg.date, datatype.DATE_TIME_FORMAT))
self.__add_ohlc_data(timestamp, msg.open,msg.high,msg.low,msg.close,msg.volume,msg.count)
def __add_ohlc_data(self, timestamp, op, hi ,lo,close,vol,cnt):
self.ohlc.loc[timestamp, "open"] = float(op)
self.ohlc.loc[timestamp, "high"] = float(hi)
self.ohlc.loc[timestamp, "low"] = float(lo)
self.ohlc.loc[timestamp, "close"] = float(close)
self.ohlc.loc[timestamp, "volume"] = float(vol)
self.ohlc.loc[timestamp, "count"] = float(cnt)
#
def __run_indicators(self, ohlc):
#hardcoding ML munging parameters now
ohlc['returns']=ta.ROC(np.asarray(ohlc['close']).astype(float))
ohlc['sma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 10)
ohlc['lma']=ta.SMA(np.asarray(ohlc['close']).astype(float), 120)
ohlc['rsi']=ta.RSI(np.asarray(ohlc['close']).astype(float))
ohlc['atr']=ta.ATR(np.asarray(ohlc['high']).astype(float),np.asarray(ohlc['low']).astype(float),np.asarray(ohlc['close']).astype(float),10)
ohlc['monday'] = np.where(ohlc.index.weekday == 0,1,0)
#bellow is because I don't know how to np.where with multiple conditions
ohlc['roll'] = np.where(ohlc.index.month % 3 == 0,1,0)
#hackish as balls:
# ohlc.index = ohlc.index.tz_localize("Singapore")
ohlc["busy"] = np.where(ohlc.index.tz_convert("America/Chicago").hour >= 9,np.where(ohlc.index.tz_convert("America/Chicago").hour <= 14,1,0),0)
def update_norm_params(self):
try:
prices = self.handler.prices["price"]
except:
self.test_logger.error("getting price from handler failed - update norm/hft")
sgp_tz = pytz.timezone('Singapore')
try:
prices.index = prices.index.tz_localize(sgp_tz)
#prices.to_pick (os.path.join(os.path.curdir,"prices_f_norm.csv"))
except:
self.test_logger.error("prices localization failed - update norm/hft")
#
try:
prices = prices.dropna()
except:
self.test_logger.error("dropna failed - update norm/hft")
try:
if prices.index.max()-prices.index.min() > dt.timedelta(seconds=60) and self.last_trim is not None:
prices = prices[prices.index > prices.index.max()-dt.timedelta(seconds=60)]#todo no trim of prices as of now
print "trimmed prices"
print prices
#prices.to_pickle(os.path.join(os.path.curdir(), "prices.pkl"))
except:
self.test_logger.error("minute trim of prices failed - update norm/hft")
if len(prices) !=0:
last_price = prices.iloc[-1]
try:
my_cur_mean = round(np.mean(prices),2)
self.handler.cur_mean = my_cur_mean
except:
self.test_logger.error("mean update failed- update norm/hft")
try:
prices = prices.diff()
prices = prices.dropna()
prices = prices**2
except:
self.test_logger.error("diff/square failed- update norm/hft")
tdiffs = list()
try:
for i in range(1,len(prices)):
tdiffs.append((prices.index[i]-prices.index[i-1]).total_seconds())
prices = prices.ix[1:]
print prices
except:
self.test_logger.error("time diffs creation failed - update norm/hft")
try:
my_cur_sd = round(sqrt(sum(prices * tdiffs)/len(prices)), 2)
self.handler.cur_sd = my_cur_sd
except:
self.test_logger.error("StDev udpate failed - update norm/hft")
self.handler.cur_sd = 2 * settings.STOP_OFFSET # in case the stdev failed
self.test_logger.error("update norm parameters completed - update norm/hft")
self.last_trim = self.now
def __cancel_market_data_request(self):
self.conn.cancelMktData(1)
time.sleep(1)
def spawn(self):
print "execution thread spawned"
self.handler = ExecutionHandler(self.conn)
def start(self, symbols):
self.test_logger.info("Started Requests !!!!")
self.conn.connect() # Get IB connection object
self.__init_stocks_data(symbols)
self.test_logger.info("Init Stock")
self.test_logger.info("Request Market Data")
self.__request_streaming_data(self.conn)
self.test_logger.info("Request Position")
self.conn.reqPositions()
self.test_logger.info("Request Historicals")
self.__request_historical_data(self.conn)
time.sleep(1)
if self.handler.position !=0:
self.test_logger.info("Squaring off for a clean start")
self.handler.neutralize()
try:
#self.time_keeper()
time.sleep(5)
self.test_logger.info("I hope Ihave ohlc now, from hft")
print self.ohlc.tail(5)
self.test_logger.info("call ML first time - HFT")
self.flag = self.ml.call_ml(self.ohlc)
self.handler.flag = self.flag # this is stupid
time.sleep(3)
print "I believe we will "+ self.flag
self.test_logger.info("Spawn concurrent processes")
self.timekeeper = self.executor.submit(self.time_keeper)
self.test_logger.info("Time keeper spawned")
time.sleep(1)
self.parser = self.executor.submit(self.handler.queue_parser)
self.test_logger.info("Parser spawned")
time.sleep(5)
self.update_norm_params()
self.test_logger.info("First normalized parameters passed")
time.sleep(5)
self.test_logger.info("Spawning strategy handler")
self.strategy = self.executor.submit(self.handler.trading_loop)
self.test_logger.info("Spawning execution handler")
self.execution = self.executor.submit(self.handler.order_execution)
except (Exception, KeyboardInterrupt):
print "Exception:"
print "Cancelling...",
self.__cancel_market_data_request()
print "killing all orders"
self.handler.kill_em_all()
# self.monitor.close_stream()
print "Disconnecting..."
time.sleep(5)
self.conn.disconnect()
time.sleep(1)
print "Disconnected."
def stop(self):
os.remove(os.path.normpath(os.path.join(os.path.curdir,"data.csv")))
os.remove(os.path.normpath(os.path.join(os.path.curdir,"ohlc.csv")))
self.__cancel_market_data_request()
#self.monitor.close_stream()
print "Disconnecting..."
self.conn.disconnect()
# self.store.close()
def rekindle_execution(self):
self.execution = self.executor.submit(self.handler.trading_loop)
