def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()
).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {arg: val for arg, val in locals().items() if arg not in ('__class__', 'self', 'kwargs')}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(
host = str(self.blotter_args['dbhost']),
port = int(self.blotter_args['dbport']),
user = str(self.blotter_args['dbuser']),
passwd = str(self.blotter_args['dbpass']),
db = str(self.blotter_args['dbname']),
autocommit = True
)
self.dbcurr = self.dbconn.cursor()
def __init__(self, instruments, resolution, \
tick_window=1, bar_window=100, timezone="UTC", preload=None, \
continuous=True, blotter=None, **kwargs):
self.name = str(self.__class__).split('.')[-1].split("'")[0]
# assign algo params
self.bars = pd.DataFrame()
self.ticks = pd.DataFrame()
self.quotes = {}
self.tick_count = 0
self.tick_bar_count = 0
self.bar_count = 0
self.bar_hash = 0
self.tick_window = tick_window if tick_window > 0 else 1
if "V" in resolution:
self.tick_window = 1000
self.bar_window = bar_window if bar_window > 0 else 100
self.resolution = resolution.replace("MIN", "T")
self.timezone = timezone
self.preload = preload
self.continuous = continuous
self.backtest = args.backtest
self.backtest_start = args.start
self.backtest_end = args.end
# -----------------------------------
self.sms_numbers = [] if args.sms is None else args.sms
self.trade_log_dir = args.log
self.blotter_name = args.blotter if args.blotter is not None else blotter
self.record_output = args.output
# -----------------------------------
# load blotter settings && initilize Blotter
self.load_blotter_args(args.blotter)
self.blotter = Blotter(**self.blotter_args)
# -----------------------------------
# initiate broker/order manager
super().__init__(instruments, ibclient=int(args.ibclient), \
ibport=int(args.ibport), ibserver=str(args.ibserver))
# -----------------------------------
# signal collector
self.signals = {}
for sym in self.symbols:
self.signals[sym] = pd.DataFrame()
# -----------------------------------
# initilize output file
self.record_ts = None
if self.record_output:
self.datastore = tools.DataStore(args.output)
# -----------------------------------
# initiate strategy
self.on_start()
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
self.dbconn = None
self.dbcurr = None
self.host = args.host if args.host is not None else host
self.port = args.port if args.port is not None else port
self.blotter_name = args.blotter if args.blotter is not None else blotter
self.load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {arg: val for arg, val in locals().items(
) if arg not in ('__class__', 'self', 'kwargs')}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(
host = str(self.blotter_args['dbhost']),
port = int(self.blotter_args['dbport']),
user = str(self.blotter_args['dbuser']),
passwd = str(self.blotter_args['dbpass']),
db = str(self.blotter_args['dbname']),
autocommit = True
)
self.dbcurr = self.dbconn.cursor()
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"):
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# initilize class logger
self.log_broker = logging.getLogger(__name__)
# default params (overrided in algo)
self.timezone = "UTC"
self.last_price = {}
self.tick_window = 1000
self.bar_window = 100
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
else:
instrument = tools.create_ib_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except Exception as e:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
self.instrument_combos = {}
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(
by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={}
)
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = load_blotter_args(
self.blotter_name, logger=self.log_broker)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
if not self.blotter_args['dbskip']:
self.dbconn = pymysql.connect(
host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True
)
self.dbcurr = self.dbconn.cursor()
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
class Broker():
"""Broker class initilizer (abstracted, parent class of ``Algo``)
:Parameters:
instruments : list
List of IB contract tuples
ibclient : int
IB TWS/GW Port to use (default: 4001)
ibport : int
IB TWS/GW Client ID (default: 998)
ibserver : string
IB TWS/GW Server hostname (default: localhost)
"""
__metaclass__ = ABCMeta
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"):
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# initilize class logger
self.log_broker = logging.getLogger(__name__)
# default params (overrided in algo)
self.timezone = "UTC"
self.last_price = {}
self.tick_window = 1000
self.bar_window = 100
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
else:
instrument = tools.create_ib_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except Exception as e:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
self.instrument_combos = {}
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(
by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={}
)
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = load_blotter_args(
self.blotter_name, logger=self.log_broker)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
if not self.blotter_args['dbskip']:
self.dbconn = pymysql.connect(
host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True
)
self.dbcurr = self.dbconn.cursor()
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
# ---------------------------------------
def add_instruments(self, *instruments):
""" add instruments after initialization """
for instrument in instruments:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
self.instruments[contractString] = instrument
self.ibConn.createContract(instrument)
self.symbols = list(self.instruments.keys())
# ---------------------------------------
@abstractmethod
def on_fill(self, instrument, order):
pass
# ---------------------------------------
"""
instrument group methods
used with spreads to get the group members (contratc legs) as symbols
"""
def register_combo(self, parent, legs):
""" add contracts to groups """
parent = self.ibConn.contractString(parent)
legs_dict = {}
for leg in legs:
leg = self.ibConn.contractString(leg)
legs_dict[leg] = self.get_instrument(leg)
self.instrument_combos[parent] = legs_dict
def get_combo(self, symbol):
""" get group by child symbol """
for parent, legs in self.instrument_combos.items():
if symbol == parent or symbol in legs.keys():
return {
"parent": self.get_instrument(parent),
"legs": legs,
}
return {
"parent": None,
"legs": {},
}
# -------------------------------------------
def _on_exit(self):
self.log_broker.info("Algo stopped...")
if self.ibConn is not None:
self.log_broker.info("Disconnecting...")
self.ibConn.disconnect()
self.log_broker.info("Disconnecting from MySQL...")
try:
self.dbcurr.close()
self.dbconn.close()
except Exception as e:
pass
# ---------------------------------------
def ibConnect(self):
self.ibConn.connect(clientId=self.ibclient,
host=self.ibserver, port=self.ibport)
self.ibConn.requestPositionUpdates(subscribe=True)
self.ibConn.requestAccountUpdates(subscribe=True)
# ---------------------------------------
# @abstractmethod
def ibCallback(self, caller, msg, **kwargs):
if caller == "handleHistoricalData":
# transmit "as-is" to blotter for handling
self.blotter.ibCallback("handleHistoricalData", msg, **kwargs)
if caller == "handleConnectionClosed":
self.log_broker.info("Lost conncetion to Interactive Brokers...")
while not self.ibConn.connected:
self.ibConnect()
time.sleep(1.3)
if not self.ibConn.connected:
print('*', end="", flush=True)
self.log_broker.info("Connection established...")
elif caller == "handleOrders":
if not hasattr(self, "orders"):
return
if msg.typeName == ezibpy.utils.dataTypes["MSG_TYPE_OPEN_ORDER_END"]:
return
# order canceled? do some cleanup
if hasattr(msg, 'status') and "CANCELLED" in msg.status.upper():
if msg.orderId in self.orders.recent.keys():
symbol = self.orders.recent[msg.orderId]['symbol']
try:
del self.orders.pending_ttls[msg.orderId]
except Exception as e:
pass
try:
del self.orders.recent[msg.orderId]
except Exception as e:
pass
try:
if self.orders.pending[symbol]['orderId'] == msg.orderId:
del self.orders.pending[symbol]
except Exception as e:
pass
return
# continue...
order = self.ibConn.orders[msg.orderId]
# print("***********************\n\n", order, "\n\n***********************")
orderId = msg.orderId
symbol = order["symbol"]
try:
try:
quantity = self.orders.history[symbol][orderId]['quantity']
except Exception as e:
quantity = self.orders.history[symbol][order['parentId']]['quantity']
# ^^ for child orders auto-created by ezibpy
except Exception as e:
quantity = 1
# update pending order to the time actually submitted
if order["status"] in ["OPENED", "SUBMITTED"]:
if orderId in self.orders.pending_ttls:
self._update_pending_order(symbol, orderId,
self.orders.pending_ttls[orderId], quantity)
elif order["status"] == "FILLED":
self._update_order_history(
symbol, orderId, quantity, filled=True)
self._expire_pending_order(symbol, orderId)
self._cancel_orphan_orders(orderId)
self._register_trade(order)
# filled
time.sleep(0.005)
self.on_fill(self.get_instrument(order['symbol']), order)
# ---------------------------------------
def _register_trade(self, order):
""" constructs trade info from order data """
if order['id'] in self.orders.recent:
orderId = order['id']
else:
orderId = order['parentId']
# entry / exit?
symbol = order["symbol"]
order_data = self.orders.recent[orderId]
position = self.get_positions(symbol)['position']
if position != 0:
# entry
order_data['action'] = "ENTRY"
order_data['position'] = position
order_data['entry_time'] = tools.datetime_to_timezone(
order['time'])
order_data['exit_time'] = None
order_data['entry_order'] = order_data['order_type']
order_data['entry_price'] = order['avgFillPrice']
order_data['exit_price'] = 0
order_data['exit_reason'] = None
else:
order_data['action'] = "EXIT"
order_data['position'] = 0
order_data['exit_time'] = tools.datetime_to_timezone(order['time'])
order_data['exit_price'] = order['avgFillPrice']
# target / stop?
if order['id'] == order_data['targetOrderId']:
order_data['exit_reason'] = "TARGET"
elif order['id'] == order_data['stopOrderId']:
order_data['exit_reason'] = "STOP"
else:
order_data['exit_reason'] = "SIGNAL"
# remove from collection
del self.orders.recent[orderId]
if order_data is None:
return None
# trade identifier
tradeId = self.strategy.upper() + '_' + symbol.upper()
tradeId = hashlib.sha1(tradeId.encode()).hexdigest()
# existing trade?
if tradeId not in self.active_trades:
self.active_trades[tradeId] = {
"strategy": self.strategy,
"action": order_data['action'],
"quantity": abs(order_data['position']),
"position": order_data['position'],
"symbol": order_data["symbol"].split('_')[0],
"direction": order_data['direction'],
"entry_time": None,
"exit_time": None,
"duration": "0s",
"exit_reason": order_data['exit_reason'],
"order_type": order_data['order_type'],
"market_price": order_data['price'],
"target": order_data['target'],
"stop": order_data['initial_stop'],
"entry_price": 0,
"exit_price": order_data['exit_price'],
"realized_pnl": 0
}
if "entry_time" in order_data:
self.active_trades[tradeId]["entry_time"] = order_data['entry_time']
if "entry_price" in order_data:
self.active_trades[tradeId]["entry_price"] = order_data['entry_price']
else:
# self.active_trades[tradeId]['direction'] = order_data['direction']
self.active_trades[tradeId]['action'] = order_data['action']
self.active_trades[tradeId]['position'] = order_data['position']
self.active_trades[tradeId]['exit_price'] = order_data['exit_price']
self.active_trades[tradeId]['exit_reason'] = order_data['exit_reason']
self.active_trades[tradeId]['exit_time'] = order_data['exit_time']
# calculate trade duration
try:
delta = int((self.active_trades[tradeId]['exit_time'] -
self.active_trades[tradeId]['entry_time']).total_seconds())
days, remainder = divmod(delta, 86400)
hours, remainder = divmod(remainder, 3600)
minutes, seconds = divmod(remainder, 60)
duration = ('%sd %sh %sm %ss' %
(days, hours, minutes, seconds))
self.active_trades[tradeId]['duration'] = duration.replace(
"0d ", "").replace("0h ", "").replace("0m ", "")
except Exception as e:
pass
trade = self.active_trades[tradeId]
if trade['entry_price'] > 0 and trade['position'] == 0:
if trade['direction'] == "SELL":
pnl = trade['entry_price'] - trade['exit_price']
else:
pnl = trade['exit_price'] - trade['entry_price']
pnl = tools.to_decimal(pnl)
# print("1)", pnl)
self.active_trades[tradeId]['realized_pnl'] = pnl
# print("\n\n-----------------")
# print(self.active_trades[tradeId])
# print("-----------------\n\n")
# get trade
trade = self.active_trades[tradeId].copy()
# sms trades
sms._send_trade(trade, self.sms_numbers, self.timezone)
# rename trade direction
trade['direction'] = trade['direction'].replace(
"BUY", "LONG").replace("SELL", "SHORT")
# log
self.log_trade(trade)
# remove from active trades and add to trade
if trade['action'] == "EXIT":
del self.active_trades[tradeId]
self.trades.append(trade)
# return trade
return trade
# ---------------------------------------
def log_trade(self, trade):
# first trade is an exit?
if trade['entry_time'] is None:
return
# connection established
if (self.dbconn is not None) & (self.dbcurr is not None):
sql = """INSERT INTO trades (
`algo`, `symbol`, `direction`,`quantity`,
`entry_time`, `exit_time`, `exit_reason`,
`order_type`, `market_price`, `target`, `stop`,
`entry_price`, `exit_price`, `realized_pnl`)
VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)
ON DUPLICATE KEY UPDATE
`algo`=%s, `symbol`=%s, `direction`=%s, `quantity`=%s,
`entry_time`=%s, `exit_time`=%s, `exit_reason`=%s,
`order_type`=%s, `market_price`=%s, `target`=%s, `stop`=%s,
`entry_price`=%s, `exit_price`=%s, `realized_pnl`=%s
"""
try:
trade['entry_time'] = trade['entry_time'].strftime(
"%Y-%m-%d %H:%M:%S.%f")
except Exception as e:
pass
try:
trade['exit_time'] = trade['exit_time'].strftime(
"%Y-%m-%d %H:%M:%S.%f")
except Exception as e:
pass
# all strings
for k, v in trade.items():
if v is not None:
trade[k] = str(v)
self.dbcurr.execute(sql, (
trade['strategy'], trade['symbol'], trade['direction'], trade['quantity'],
trade['entry_time'], trade['exit_time'], trade['exit_reason'],
trade['order_type'], trade['market_price'], trade['target'], trade['stop'],
trade['entry_price'], trade['exit_price'], trade['realized_pnl'],
trade['strategy'], trade['symbol'], trade['direction'], trade['quantity'],
trade['entry_time'], trade['exit_time'], trade['exit_reason'],
trade['order_type'], trade['market_price'], trade['target'], trade['stop'],
trade['entry_price'], trade['exit_price'], trade['realized_pnl']
))
# commit
try:
self.dbconn.commit()
except Exception as e:
pass
if self.trade_log_dir:
self.trade_log_dir = (self.trade_log_dir + '/').replace('//', '/')
trade_log_path = self.trade_log_dir + self.strategy.lower() + "_" + \
datetime.now().strftime('%Y%m%d') + ".csv"
# convert None to empty string !!
trade.update((k, '') for k, v in trade.items() if v is None)
# create df
trade_df = pd.DataFrame(index=[0], data=trade)[[
'strategy', 'symbol', 'direction', 'quantity', 'entry_time',
'exit_time', 'exit_reason', 'order_type', 'market_price', 'target',
'stop', 'entry_price', 'exit_price', 'realized_pnl'
]]
if os.path.exists(trade_log_path):
trades = pd.read_csv(trade_log_path, header=0)
trades = trades.append(trade_df, ignore_index=True, sort=True)
trades.drop_duplicates(['entry_time', 'symbol', 'strategy'],
keep="last", inplace=True)
trades.to_csv(trade_log_path, header=True, index=False)
tools.chmod(trade_log_path)
else:
trade_df.to_csv(trade_log_path, header=True, index=False)
tools.chmod(trade_log_path)
# ---------------------------------------
def active_order(self, symbol, order_type="STOP"):
if symbol in self.orders.history:
for orderId in self.orders.history[symbol]:
order = self.orders.history[symbol][orderId]
if order['order_type'].upper() == order_type.upper():
return order
return None
# ---------------------------------------
@staticmethod
def _get_locals(local_params):
del local_params['self']
return local_params
# ---------------------------------------
def _create_order(self, symbol, direction, quantity, order_type="",
limit_price=0, expiry=0, orderId=0, target=0, initial_stop=0,
trail_stop_at=0, trail_stop_by=0, stop_limit=False, **kwargs):
# fix prices to comply with contract's min-tick
ticksize = self.get_contract_details(symbol)['m_minTick']
limit_price = tools.round_to_fraction(limit_price, ticksize)
target = tools.round_to_fraction(target, ticksize)
initial_stop = tools.round_to_fraction(initial_stop, ticksize)
trail_stop_at = tools.round_to_fraction(trail_stop_at, ticksize)
trail_stop_by = tools.round_to_fraction(trail_stop_by, ticksize)
self.log_broker.debug('CREATE ORDER: %s %4d %s %s', direction,
quantity, symbol, dict(locals(), **kwargs))
# force BUY/SELL (not LONG/SHORT)
direction = direction.replace("LONG", "BUY").replace("SHORT", "SELL")
# modify order?
if order_type.upper() == "MODIFY":
self.modify_order(symbol, orderId, quantity, limit_price)
return
# continue...
if "stoploss" in kwargs and initial_stop == 0:
initial_stop = kwargs['stoploss']
order_type = "MARKET" if limit_price == 0 else "LIMIT"
fillorkill = kwargs["fillorkill"] if "fillorkill" in kwargs else False
iceberg = kwargs["iceberg"] if "iceberg" in kwargs else False
tif = kwargs["tif"] if "tif" in kwargs else "DAY"
# clear expired pending orders
self._cancel_expired_pending_orders()
# don't submit order if a pending one is waiting
if symbol in self.orders.pending:
self.log_broker.warning(
'Not submitting %s order, orders pending: %s', symbol, self.orders.pending)
return
# @TODO - decide on quantity here
# continue...
order_quantity = abs(quantity)
if direction.upper() == "SELL":
order_quantity = -order_quantity
contract = self.get_contract(symbol)
# is bracket order
bracket = (target > 0) | (initial_stop > 0) | (
trail_stop_at > 0) | (trail_stop_by > 0)
# create & submit order
if not bracket:
# simple order
order = self.ibConn.createOrder(order_quantity, limit_price,
fillorkill=fillorkill,
iceberg=iceberg,
tif=tif)
orderId = self.ibConn.placeOrder(contract, order)
self.log_broker.debug('PLACE ORDER: %s %s', tools.contract_to_dict(
contract), tools.order_to_dict(order))
else:
# bracket order
order = self.ibConn.createBracketOrder(contract, order_quantity,
entry=limit_price,
target=target,
stop=initial_stop,
stop_limit=stop_limit,
fillorkill=fillorkill,
iceberg=iceberg,
tif=tif)
orderId = order["entryOrderId"]
# triggered trailing stop?
if trail_stop_by != 0 and trail_stop_at != 0:
self.ibConn.createTriggerableTrailingStop(symbol, -order_quantity,
triggerPrice=trail_stop_at,
trailPercent=trail_stop_by,
# trailAmount = trail_stop_by,
parentId=order['entryOrderId'],
stopOrderId=order["stopOrderId"]
)
# add all orders to history
self._update_order_history(symbol=symbol,
orderId=order["entryOrderId"],
quantity=order_quantity,
order_type='ENTRY')
self._update_order_history(symbol=symbol,
orderId=order["targetOrderId"],
quantity=-order_quantity,
order_type='TARGET',
parentId=order["entryOrderId"])
self._update_order_history(symbol=symbol,
orderId=order["stopOrderId"],
quantity=-order_quantity,
order_type='STOP',
parentId=order["entryOrderId"])
# have original params available for FILL event
self.orders.recent[orderId] = self._get_locals(locals())
self.orders.recent[orderId]['targetOrderId'] = 0
self.orders.recent[orderId]['stopOrderId'] = 0
if bracket:
self.orders.recent[orderId]['targetOrderId'] = order["targetOrderId"]
self.orders.recent[orderId]['stopOrderId'] = order["stopOrderId"]
# append market price at the time of order
try:
self.orders.recent[orderId]['price'] = self.last_price[symbol]
except Exception as e:
self.orders.recent[orderId]['price'] = 0
# add orderId / ttl to (auto-adds to history)
expiry = expiry * 1000 if expiry > 0 else 60000 # 1min
self._update_pending_order(symbol, orderId, expiry, order_quantity)
# ---------------------------------------
def _cancel_order(self, orderId):
if orderId is not None and orderId > 0:
self.ibConn.cancelOrder(orderId)
# ---------------------------------------
def modify_order_group(self, symbol, orderId, entry=None,
target=None, stop=None, quantity=None):
order_group = self.orders.recent[orderId]['order']
if entry is not None:
self.modify_order(
symbol, orderId, limit_price=entry, quantity=quantity)
if target is not None:
self.modify_order(symbol, order_group['targetOrderId'],
limit_price=target, quantity=quantity)
if stop is not None:
stop_quantity = quantity * -1 if quantity is not None else None
self.modify_order(symbol, order_group['stopOrderId'],
limit_price=stop, quantity=stop_quantity)
# ---------------------------------------
def modify_order(self, symbol, orderId, quantity=None, limit_price=None):
if quantity is None and limit_price is None:
return
if symbol in self.orders.history:
for historyOrderId in self.orders.history[symbol]:
if historyOrderId == orderId:
order_quantity = self.orders.history[symbol][orderId]['quantity']
if quantity is not None:
order_quantity = quantity
order = self.orders.history[symbol][orderId]
if order['order_type'] == "STOP":
new_order = self.ibConn.createStopOrder(
quantity=order_quantity,
parentId=order['parentId'],
stop=limit_price,
trail=None,
transmit=True
)
else:
new_order = self.ibConn.createOrder(
order_quantity, limit_price)
# child order?
if "parentId" in order:
new_order.parentId = order['parentId']
# send order
contract = self.get_contract(symbol)
self.ibConn.placeOrder(
contract, new_order, orderId=orderId)
break
# ---------------------------------------
@staticmethod
def _milliseconds_delta(delta):
return delta.days * 86400000 + delta.seconds * 1000 + delta.microseconds / 1000
# ---------------------------------------
def _cancel_orphan_orders(self, orderId):
""" cancel child orders when parent is gone """
orders = self.ibConn.orders
for order in orders:
order = orders[order]
if order['parentId'] != orderId:
self.ibConn.cancelOrder(order['id'])
# ---------------------------------------
def _cancel_expired_pending_orders(self):
""" expires pending orders """
# use a copy to prevent errors
pending = self.orders.pending.copy()
for symbol in pending:
orderId = pending[symbol]["orderId"]
expiration = pending[symbol]["expires"]
delta = expiration - datetime.now()
delta = self._milliseconds_delta(delta)
# cancel order if expired
if delta < 0:
self.ibConn.cancelOrder(orderId)
if orderId in self.orders.pending_ttls:
if orderId in self.orders.pending_ttls:
del self.orders.pending_ttls[orderId]
if symbol in self.orders.pending:
if self.orders.pending[symbol]['orderId'] == orderId:
del self.orders.pending[symbol]
# ---------------------------------------------------------
def _expire_pending_order(self, symbol, orderId):
self.ibConn.cancelOrder(orderId)
if orderId in self.orders.pending_ttls:
del self.orders.pending_ttls[orderId]
if symbol in self.orders.pending:
if self.orders.pending[symbol]['orderId'] == orderId:
del self.orders.pending[symbol]
# ---------------------------------------------------------
def _update_pending_order(self, symbol, orderId, expiry, quantity):
self.orders.pending[symbol] = {
"orderId": orderId,
"quantity": quantity,
# "created": datetime.now(),
"expires": datetime.now() + timedelta(milliseconds=expiry)
}
# ibCallback needs this to update with submittion time
self.orders.pending_ttls[orderId] = expiry
self._update_order_history(
symbol=symbol, orderId=orderId, quantity=quantity)
# ---------------------------------------------------------
def _update_order_history(self, symbol, orderId, quantity,
order_type='entry', filled=False, parentId=0):
if symbol not in self.orders.history:
self.orders.history[symbol] = {}
self.orders.history[symbol][orderId] = {
"orderId": orderId,
"quantity": quantity,
"order_type": order_type.upper(),
"filled": filled,
"parentId": parentId
}
# ---------------------------------------
# UTILITY FUNCTIONS
# ---------------------------------------
def get_instrument(self, symbol):
"""
A string subclass that provides easy access to misc
symbol-related methods and information using shorthand.
Refer to the `Instruments API <#instrument-api>`_
for available methods and properties
Call from within your strategy:
``instrument = self.get_instrument("SYMBOL")``
:Parameters:
symbol : string
instrument symbol
"""
instrument = Instrument(self.get_symbol(symbol))
instrument._set_parent(self)
instrument._set_windows(ticks=self.tick_window, bars=self.bar_window)
return instrument
# ---------------------------------------
@staticmethod
def get_symbol(symbol):
if not isinstance(symbol, str):
if isinstance(symbol, dict):
symbol = symbol['symbol']
elif isinstance(symbol, pd.DataFrame):
symbol = symbol[:1]['symbol'].values[0]
return symbol
# ---------------------------------------
def get_account(self):
return self.ibConn.account
# ---------------------------------------
def get_contract(self, symbol):
return self.ibConn.contracts[self.ibConn.tickerId(symbol)]
# ---------------------------------------
def get_contract_details(self, symbol):
return self.ibConn.contractDetails(symbol)
# ---------------------------------------
def get_tickerId(self, symbol):
return self.ibConn.tickerId(symbol)
# ---------------------------------------
def get_orders(self, symbol):
symbol = self.get_symbol(symbol)
self.orders.by_symbol = self.ibConn.group_orders("symbol")
if symbol in self.orders.by_symbol:
return self.orders.by_symbol[symbol]
return {}
# ---------------------------------------
def get_positions(self, symbol):
symbol = self.get_symbol(symbol)
if symbol in self.ibConn.positions:
return self.ibConn.positions[symbol]
return {
"symbol": symbol,
"position": 0,
"avgCost": 0.0,
"account": None
}
# ---------------------------------------
def get_portfolio(self, symbol=None):
if symbol is not None:
symbol = self.get_symbol(symbol)
if symbol in self.ibConn.portfolio:
portfolio = self.ibConn.portfolio[symbol]
if "symbol" in portfolio:
return portfolio
return {
"symbol": symbol,
"position": 0.0,
"marketPrice": 0.0,
"marketValue": 0.0,
"averageCost": 0.0,
"unrealizedPNL": 0.0,
"realizedPNL": 0.0,
"totalPNL": 0.0,
"account": None
}
return self.ibConn.portfolio
# ---------------------------------------
def get_pending_orders(self, symbol=None):
if symbol is not None:
symbol = self.get_symbol(symbol)
if symbol in self.orders.pending:
return self.orders.pending[symbol]
return {}
return self.orders.pending
# ---------------------------------------
def get_trades(self, symbol=None):
# closed trades
trades = pd.DataFrame(self.trades)
if not trades.empty:
trades.loc[:, 'closed'] = True
# ongoing trades
active_trades = pd.DataFrame(list(self.active_trades.values()))
if not active_trades.empty:
active_trades.loc[:, 'closed'] = False
# combine dataframes
df = pd.concat([trades, active_trades], sort=True).reset_index()
# set last price
if not df.empty:
# conert values to floats
df['entry_price'] = df['entry_price'].astype(float)
df['exit_price'] = df['exit_price'].astype(float)
df['market_price'] = df['market_price'].astype(float)
df['realized_pnl'] = df['realized_pnl'].astype(float)
df['stop'] = df['stop'].astype(float)
df['target'] = df['target'].astype(float)
df['quantity'] = df['quantity'].astype(int)
try:
df.loc[:, 'last'] = self.last_price[symbol]
except Exception as e:
df.loc[:, 'last'] = 0
# calc unrealized pnl
df['unrealized_pnl'] = np.where(df['direction'] == "SHORT",
df['entry_price'] - df['last'],
df['last'] - df['entry_price'])
df.loc[df['closed'], 'unrealized_pnl'] = 0
# drop index column
df.drop('index', axis=1, inplace=True)
# get single symbol
if symbol is not None:
df = df[df['symbol'] == symbol.split("_")[0]]
df.loc[:, 'symbol'] = symbol
# return
return df
class Reports():
"""Reports class initilizer
:Optional:
blotter : str
Log trades to MySQL server used by this Blotter (default is "auto detect")
port : int
HTTP port to use (default: 5000)
host : string
Host to bind the http process to (defaults 0.0.0.0)
password : string
Password for logging in to the web app (auto-generated by default). Use "" for no password.
"""
# ---------------------------------------
def __init__(self,
blotter=None,
port=5000,
host="0.0.0.0",
password=None,
nopass=False,
**kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {
arg: val
for arg, val in locals().items()
if arg not in ('__class__', 'self', 'kwargs')
}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args[
'blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True)
self.dbcurr = self.dbconn.cursor()
# ---------------------------------------
def load_cli_args(self):
"""
Parse command line arguments and return only the non-default ones
:Retruns: dict
a dict of any non-default args passed on the command-line.
"""
parser = argparse.ArgumentParser(
description='QTPyLib Reporting',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--port',
default=self.args["port"],
help='HTTP port to use',
type=int)
parser.add_argument('--host',
default=self.args["host"],
help='Host to bind the http process to')
parser.add_argument('--blotter',
help='Use this Blotter\'s MySQL server settings')
parser.add_argument('--nopass',
help='Skip password for web app (flag)',
action='store_true')
# only return non-default cmd line args
# (meaning only those actually given)
cmd_args, unknown = parser.parse_known_args()
args = {
arg: val
for arg, val in vars(cmd_args).items()
if val != parser.get_default(arg)
}
return args
# ---------------------------------------
def send_static(self, path):
return send_from_directory('_webapp/', path)
# ---------------------------------------
def login(self, password):
if self._password == password:
resp = make_response('yes')
resp.set_cookie('password', password)
return resp
else:
return make_response("no")
# ---------------------------------------
def algos(self, json=True):
algos = pd.read_sql("SELECT DISTINCT algo FROM trades",
self.dbconn).to_dict(orient="records")
if json:
return jsonify(algos)
else:
return algos
# ---------------------------------------
def symbols(self, json=True):
symbols = pd.read_sql("SELECT * FROM symbols",
self.dbconn).to_dict(orient="records")
if json:
return jsonify(symbols)
else:
return symbols
# ---------------------------------------
def trades(self, start=None, end=None, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
trades_query = "SELECT * FROM trades WHERE exit_time IS NOT NULL"
trades_where = []
if isinstance(start, str):
trades_where.append("entry_time>='" + start + "'")
if isinstance(end, str):
trades_where.append("exit_time<='" + end + "'")
if algo_id is not None:
trades_where.append("algo='" + algo_id + "'")
if len(trades_where) > 0:
trades_query += " AND " + " AND ".join(trades_where)
trades = pd.read_sql(trades_query, self.dbconn)
trades['exit_time'].fillna(0, inplace=True)
trades['slippage'] = abs(trades['entry_price'] -
trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") &
(trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") &
(trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['exit_time', 'entry_time'],
ascending=[False, False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def positions(self, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
trades_query = "SELECT * FROM trades WHERE exit_time IS NULL"
if algo_id is not None:
trades_query += " AND algo='" + algo_id + "'"
trades = pd.read_sql(trades_query, self.dbconn)
last_query = "SELECT s.id, s.symbol, max(t.last) as last_price FROM ticks t LEFT JOIN symbols s ON (s.id=t.symbol_id) GROUP BY s.id"
last_prices = pd.read_sql(last_query, self.dbconn)
trades = trades.merge(last_prices, on=['symbol'])
trades['unrealized_pnl'] = np.where(
trades['direction'] == "SHORT",
trades['entry_price'] - trades['last_price'],
trades['last_price'] - trades['entry_price'])
trades['slippage'] = abs(trades['entry_price'] -
trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") &
(trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") &
(trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['entry_time'], ascending=[False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def trades_by_algo(self, algo_id=None, start=None, end=None):
trades = self.trades(start, end, algo_id=algo_id, json=False)
return jsonify(trades)
# ---------------------------------------
def bars(self, resolution, symbol, start=None, end=None, json=True):
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
bars = self.blotter.history(symbols=symbol,
start=start,
end=end,
resolution=resolution)
bars['datetime'] = bars.index
bars = bars.to_dict(orient="records")
if json:
return jsonify(bars)
else:
return bars
# ---------------------------------------
def index(self, start=None, end=None):
if not self.args['nopass']:
if self._password != "" and self._password != request.cookies.get(
'password'):
return render_template('login.html')
return render_template('dashboard.html')
# ---------------------------------------
def run(self):
"""Starts the reporting module
Makes the dashboard web app available via localhost:port, and exposes
a REST API for trade information, open positions and market data.
"""
global app
# -----------------------------------
# assign view
app.add_url_rule('/', 'index', view_func=self.index)
app.add_url_rule('/<path:start>', 'index', view_func=self.index)
app.add_url_rule('/<start>/<path:end>', 'index', view_func=self.index)
app.add_url_rule('/algos', 'algos', view_func=self.algos)
app.add_url_rule('/symbols', 'symbols', view_func=self.symbols)
app.add_url_rule('/positions', 'positions', view_func=self.positions)
app.add_url_rule('/positions/<path:algo_id>',
'positions',
view_func=self.positions)
app.add_url_rule('/algo/<path:algo_id>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<path:start>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<start>/<path:end>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/bars/<resolution>/<symbol>',
'bars',
view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<path:start>',
'bars',
view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<start>/<path:end>',
'bars',
view_func=self.bars)
app.add_url_rule('/trades', 'trades', view_func=self.trades)
app.add_url_rule('/trades/<path:start>',
'trades',
view_func=self.trades)
app.add_url_rule('/trades/<start>/<path:end>',
'trades',
view_func=self.trades)
app.add_url_rule('/login/<password>', 'login', view_func=self.login)
app.add_url_rule('/static/<path>',
'send_static',
view_func=self.send_static)
# let user know what the temp password is
if not self.args['nopass'] and self._password != "":
print(" * Web app password is:", self._password)
# notice
# print(" * Running on http://"+ str(self.host) +":"+str(self.port)+"/ (Press CTRL+C to quit)")
# -----------------------------------
# run flask app
app.run(debug=True, host=str(self.host), port=int(self.port))
def __init__(self, instruments, resolution,
tick_window=1, bar_window=100, timezone="UTC", preload=None,
continuous=True, blotter=None, force_resolution=False, **kwargs):
self.name = str(self.__class__).split('.')[-1].split("'")[0]
# ----------------------------------------------------
# default args
self.args = kwargs.copy()
cli_args = self.load_cli_args()
# override kwargs args with cli args
for arg in cli_args:
if arg not in self.args or ( arg in self.args and cli_args[arg] is not None ):
self.args[arg] = cli_args[arg]
# fix flag args (no value)
for arg in ["backtest"]:
if arg in kwargs and "--"+str(arg) not in sys.argv:
self.args[arg] = kwargs["backtest"]
# ----------------------------------------------------
# assign algo params
self.bars = pd.DataFrame()
self.ticks = pd.DataFrame()
self.quotes = {}
self.books = {}
self.tick_count = 0
self.tick_bar_count = 0
self.bar_count = 0
self.bar_hash = 0
self.tick_window = tick_window if tick_window > 0 else 1
if "V" in resolution:
self.tick_window = 1000
self.bar_window = bar_window if bar_window > 0 else 100
self.resolution = resolution.replace("MIN", "T")
self.timezone = timezone
self.preload = preload
self.continuous = continuous
self.backtest = self.args["backtest"]
self.backtest_start = self.args["start"]
self.backtest_end = self.args["end"]
# -----------------------------------
self.sms_numbers = [] if self.args["sms"] is None else self.args["sms"]
self.trade_log_dir = self.args["log"]
self.blotter_name = self.args["blotter"] if self.args["blotter"] is not None else blotter
self.record_output = self.args["output"]
# -----------------------------------
# load blotter settings && initilize Blotter
self.load_blotter_args(self.args["blotter"])
self.blotter = Blotter(**self.blotter_args)
# -----------------------------------
# initiate broker/order manager
super().__init__(instruments, ibclient=int(self.args["ibclient"]),
ibport=int(self.args["ibport"]), ibserver=str(self.args["ibserver"]))
# -----------------------------------
# signal collector
self.signals = {}
for sym in self.symbols:
self.signals[sym] = pd.DataFrame()
# -----------------------------------
# initilize output file
self.record_ts = None
if self.record_output:
self.datastore = tools.DataStore(self.args["output"])
# -----------------------------------
# initiate strategy
self.on_start()
# ---------------------------------------
# add stale ticks to allow for interval-based bars
if force_resolution and self.resolution[-1] not in ("K", "V"):
self.bar_timer = tools.RecurringTask(
self.add_stale_tick, interval_sec=1, init_sec=1, daemon=True)
class Algo(Broker):
"""Algo class initilizer (sub-class of Broker)
:Parameters:
instruments : list
List of IB contract tuples
resolution : str
Desired bar resolution (using pandas resolution: 1T, 1H, etc). Use K for tick bars.
tick_window : int
Length of tick lookback window to keep. Defaults to 1
bar_window : int
Length of bar lookback window to keep. Defaults to 100
timezone : str
Convert IB timestamps to this timezone (eg. US/Central). Defaults to UTC
preload : str
Preload history when starting algo (using pandas resolution: 1H, 1D, etc). Use K for tick bars.
continuous : bool
Tells preloader to construct continuous Futures contracts (default is True)
blotter : str
Log trades to MySQL server used by this Blotter (default is "auto detect")
force_resolution : bool
Force new bar on every ``resolution`` even if no new ticks received (default is False)
"""
__metaclass__ = ABCMeta
def __init__(self, instruments, resolution,
tick_window=1, bar_window=100, timezone="UTC", preload=None,
continuous=True, blotter=None, force_resolution=False, **kwargs):
self.name = str(self.__class__).split('.')[-1].split("'")[0]
# ----------------------------------------------------
# default args
self.args = kwargs.copy()
cli_args = self.load_cli_args()
# override kwargs args with cli args
for arg in cli_args:
if arg not in self.args or ( arg in self.args and cli_args[arg] is not None ):
self.args[arg] = cli_args[arg]
# fix flag args (no value)
for arg in ["backtest"]:
if arg in kwargs and "--"+str(arg) not in sys.argv:
self.args[arg] = kwargs["backtest"]
# ----------------------------------------------------
# assign algo params
self.bars = pd.DataFrame()
self.ticks = pd.DataFrame()
self.quotes = {}
self.books = {}
self.tick_count = 0
self.tick_bar_count = 0
self.bar_count = 0
self.bar_hash = 0
self.tick_window = tick_window if tick_window > 0 else 1
if "V" in resolution:
self.tick_window = 1000
self.bar_window = bar_window if bar_window > 0 else 100
self.resolution = resolution.replace("MIN", "T")
self.timezone = timezone
self.preload = preload
self.continuous = continuous
self.backtest = self.args["backtest"]
self.backtest_start = self.args["start"]
self.backtest_end = self.args["end"]
# -----------------------------------
self.sms_numbers = [] if self.args["sms"] is None else self.args["sms"]
self.trade_log_dir = self.args["log"]
self.blotter_name = self.args["blotter"] if self.args["blotter"] is not None else blotter
self.record_output = self.args["output"]
# -----------------------------------
# load blotter settings && initilize Blotter
self.load_blotter_args(self.args["blotter"])
self.blotter = Blotter(**self.blotter_args)
# -----------------------------------
# initiate broker/order manager
super().__init__(instruments, ibclient=int(self.args["ibclient"]),
ibport=int(self.args["ibport"]), ibserver=str(self.args["ibserver"]))
# -----------------------------------
# signal collector
self.signals = {}
for sym in self.symbols:
self.signals[sym] = pd.DataFrame()
# -----------------------------------
# initilize output file
self.record_ts = None
if self.record_output:
self.datastore = tools.DataStore(self.args["output"])
# -----------------------------------
# initiate strategy
self.on_start()
# ---------------------------------------
# add stale ticks to allow for interval-based bars
if force_resolution and self.resolution[-1] not in ("K", "V"):
self.bar_timer = tools.RecurringTask(
self.add_stale_tick, interval_sec=1, init_sec=1, daemon=True)
# ---------------------------------------
def add_stale_tick(self):
if len(self.ticks) > 0:
tick = self.ticks[-1:].to_dict(orient='records')[0]
tick['timestamp'] = datetime.utcnow()
tick = pd.DataFrame(index=[0], data=tick)
tick.set_index('timestamp', inplace=True)
tick = tools.set_timezone(tick, tz=self.timezone)
self._tick_handler(tick, stale_tick=True)
# ---------------------------------------
def load_cli_args(self):
parser = argparse.ArgumentParser(description='QTPy Algo Framework')
parser.add_argument('--ibport', default='4001', help='IB TWS/GW Port to use (default: 4001)', required=False)
parser.add_argument('--ibclient', default='998', help='IB TWS/GW Client ID (default: 998)', required=False)
parser.add_argument('--ibserver', default='localhost', help='IB TWS/GW Server hostname (default: localhost)', required=False)
parser.add_argument('--sms', nargs='+', help='Numbers to text orders', required=False)
parser.add_argument('--log', default=None, help='Path to store trade data (default: ~/qpy/trades/)', required=False)
parser.add_argument('--backtest', help='Work in Backtest mode', action='store_true', required=False)
parser.add_argument('--start', help='Backtest start date', required=False)
parser.add_argument('--end', help='Backtest end date', required=False)
parser.add_argument('--output', help='Path to save the recorded data', required=False)
parser.add_argument('--blotter', help='Log trades to the MySQL server used by this Blotter', required=False)
args, unknown = parser.parse_known_args()
return args.__dict__
# ---------------------------------------
def run(self):
"""Starts the algo
Connects to the Blotter, processes market data and passes
tick data to the ``on_tick`` function and bar data to the
``on_bar`` methods.
"""
# -----------------------------------
# backtest mode?
if self.backtest:
if self.record_output is None:
logging.error("Must provide an output file for Backtest mode")
sys.exit(0)
if self.backtest_start is None:
logging.error("Must provide start date for Backtest mode")
sys.exit(0)
if self.backtest_end is None:
self.backtest_end = datetime.now().strftime('%Y-%m-%d %H:%M:%S.%f')
# backtest history
self.blotter.drip(
symbols = self.symbols,
start = self.backtest_start,
end = self.backtest_end,
resolution = self.resolution,
tz = self.timezone,
continuous = self.continuous,
quote_handler = self._quote_handler,
tick_handler = self._tick_handler,
bar_handler = self._bar_handler,
book_handler = self._book_handler
)
# -----------------------------------
# live data mode
else:
# preload history
if self.preload is not None:
try: # dbskip may be active
self.bars = self.blotter.history(
symbols = self.symbols,
start = tools.backdate(self.preload),
resolution = self.resolution,
tz = self.timezone,
continuous = self.continuous
)
except:
pass
# print(self.bars)
# add instruments to blotter in case they do not exist
self.blotter.register(self.instruments)
# listen for RT data
self.blotter.listen(
symbols = self.symbols,
tz = self.timezone,
quote_handler = self._quote_handler,
tick_handler = self._tick_handler,
bar_handler = self._bar_handler,
book_handler = self._book_handler
)
# ---------------------------------------
@abstractmethod
def on_start(self):
"""
Invoked once when algo starts. Used for when the strategy
needs to initialize parameters upon starting.
"""
# raise NotImplementedError("Should implement on_start()")
pass
# ---------------------------------------
@abstractmethod
def on_quote(self, instrument):
"""
Invoked on every quote captured for the selected instrument.
This is where you'll write your strategy logic for quote events.
:Parameters:
symbol : string
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_quote()")
pass
# ---------------------------------------
@abstractmethod
def on_tick(self, instrument):
"""
Invoked on every tick captured for the selected instrument.
This is where you'll write your strategy logic for tick events.
:Parameters:
symbol : string
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_tick()")
pass
# ---------------------------------------
@abstractmethod
def on_bar(self, instrument):
"""
Invoked on every tick captured for the selected instrument.
This is where you'll write your strategy logic for tick events.
:Parameters:
instrument : object
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_bar()")
pass
# ---------------------------------------
@abstractmethod
def on_orderbook(self, instrument):
"""
Invoked on every change to the orderbook for the selected instrument.
This is where you'll write your strategy logic for orderbook changes events.
:Parameters:
symbol : string
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_orderbook()")
pass
# ---------------------------------------
@abstractmethod
def on_fill(self, instrument, order):
"""
Invoked on every order fill for the selected instrument.
This is where you'll write your strategy logic for fill events.
:Parameters:
instrument : object
`Instruments Object <#instrument-api>`_
order : object
Filled order data
"""
# raise NotImplementedError("Should implement on_fill()")
pass
# ---------------------------------------
def get_instrument(self, symbol):
"""
A string subclass that provides easy access to misc
symbol-related methods and information using shorthand.
Refer to the `Instruments API <#instrument-api>`_
for available methods and properties
Call from within your strategy:
``instrument = self.get_instrument("SYMBOL")``
:Parameters:
symbol : string
instrument symbol
"""
instrument = Instrument(self._getsymbol_(symbol))
instrument._set_parent(self)
return instrument
# ---------------------------------------
def get_history(self, symbols, start, end=None, resolution="1T", tz="UTC"):
"""Get historical market data.
Connects to Blotter and gets historical data from storage
:Parameters:
symbols : list
List of symbols to fetch history for
start : datetime / string
History time period start date (datetime or YYYY-MM-DD[ HH:MM[:SS]] string)
:Optional:
end : datetime / string
History time period end date (datetime or YYYY-MM-DD[ HH:MM[:SS]] string)
resolution : string
History resoluton (Pandas resample, defaults to 1T/1min)
tz : string
History timezone (defaults to UTC)
:Returns:
history : pd.DataFrame
Pandas DataFrame object with historical data for all symbols
"""
return self.blotter.history(symbols, start, end, resolution, tz)
# ---------------------------------------
# shortcuts to broker._create_order
# ---------------------------------------
def order(self, signal, symbol, quantity=0, **kwargs):
""" Send an order for the selected instrument
:Parameters:
direction : string
Order Type (BUY/SELL, EXIT/FLATTEN)
symbol : string
instrument symbol
quantity : int
Order quantiry
:Optional:
limit_price : float
In case of a LIMIT order, this is the LIMIT PRICE
expiry : int
Cancel this order if not filled after *n* seconds (default 60 seconds)
order_type : string
Type of order: Market (default), LIMIT (default when limit_price is passed), MODIFY (required passing or orderId)
orderId : int
If modifying an order, the order id of the modified order
target : float
target (exit) price
initial_stop : float
price to set hard stop
stop_limit: bool
Flag to indicate if the stop should be STOP or STOP LIMIT (default False=STOP)
trail_stop_at : float
price at which to start trailing the stop
trail_stop_by : float
% of trailing stop distance from current price
fillorkill: bool
fill entire quantiry or none at all
iceberg: bool
is this an iceberg (hidden) order
tif: str
time in force (DAY, GTC, IOC, GTD). default is ``DAY``
"""
if signal.upper() == "EXIT" or signal.upper() == "FLATTEN":
position = self.get_positions(symbol)
if position['position'] == 0:
return
kwargs['symbol'] = symbol
kwargs['quantity'] = abs(position['position'])
kwargs['direction'] = "BUY" if position['position'] < 0 else "SELL"
# print("EXIT", kwargs)
try: self.record(position=0)
except: pass
if not self.backtest:
self._create_order(**kwargs)
else:
if quantity == 0:
return
kwargs['symbol'] = symbol
kwargs['quantity'] = abs(quantity)
kwargs['direction'] = signal.upper()
# print(signal.upper(), kwargs)
# record
try:
quantity = -quantity if kwargs['direction'] == "BUY" else quantity
self.record(position=quantity)
except:
pass
if not self.backtest:
self._create_order(**kwargs)
# ---------------------------------------
def cancel_order(self, orderId):
""" Cancels a un-filled order
Parameters:
orderId : int
Order ID
"""
self._cancel_order(orderId)
# ---------------------------------------
def record(self, *args, **kwargs):
"""Records data for later analysis.
Values will be logged to the file specified via
``--output [file]`` (along with bar data) as
csv/pickle/h5 file.
Call from within your strategy:
``self.record(key=value)``
:Parameters:
** kwargs : mixed
The names and values to record
"""
if self.record_output:
try: self.datastore.record(self.record_ts, *args, **kwargs)
except: pass
# ---------------------------------------
def sms(self, text):
"""Sends an SMS message.
Relies on properly setting up an SMS provider (refer to the
SMS section of the documentation for more information about this)
Call from within your strategy:
``self.sms("message text")``
:Parameters:
text : string
The body of the SMS message to send
"""
logging.info("SMS: "+str(text))
sms.send_text(self.name +': '+ str(text), self.sms_numbers)
# ---------------------------------------
def _caller(self, caller):
stack = [x[3] for x in inspect.stack()][1:-1]
return caller in stack
# ---------------------------------------
def _book_handler(self, book):
symbol = book['symbol']
del book['symbol']
del book['kind']
self.books[symbol] = book
self.on_orderbook(self.get_instrument(symbol))
# ---------------------------------------
def _quote_handler(self, quote):
del quote['kind']
self.quotes[quote['symbol']] = quote
self.on_quote(self.get_instrument(quote))
# ---------------------------------------
def _tick_handler(self, tick, stale_tick=False):
self._cancel_expired_pending_orders()
# initial value
if self.record_ts is None:
self.record_ts = tick.index[0]
if self.resolution[-1] not in ("S", "K", "V"):
self.ticks = self._update_window(self.ticks, tick, window=self.tick_window)
else:
self.ticks = self._update_window(self.ticks, tick)
bars = tools.resample(self.ticks, self.resolution)
if len(bars.index) > self.tick_bar_count > 0 or stale_tick:
self.record_ts = tick.index[0]
self._bar_handler(bars)
periods = int("".join([s for s in self.resolution if s.isdigit()]))
self.ticks = self.ticks[-periods:]
self.tick_bar_count = len(bars)
# record tick bars
self.record(bars[-1:])
if not stale_tick:
self.on_tick(self.get_instrument(tick))
# ---------------------------------------
def _bar_handler(self, bar):
is_tick_or_volume_bar = False
handle_bar = True
if self.resolution[-1] in ("S", "K", "V"):
is_tick_or_volume_bar = True
handle_bar = self._caller("_tick_handler")
# drip is also ok
handle_bar = handle_bar or self._caller("drip")
if is_tick_or_volume_bar:
# just add a bar (used by tick bar bandler)
self.bars = self._update_window(self.bars, bar, window=self.bar_window)
else:
# add the bar and resample to resolution
self.bars = self._update_window(self.bars, bar, window=self.bar_window, resolution=self.resolution)
# new bar?
this_bar_hash = abs(hash(str(self.bars.index.values[-1]))) % (10 ** 8)
newbar = (self.bar_hash != this_bar_hash)
self.bar_hash = this_bar_hash
if newbar and handle_bar:
self.record_ts = bar.index[0]
self.on_bar(self.get_instrument(bar))
if self.resolution[-1] not in ("S", "K", "V"):
self.record(bar)
# ---------------------------------------
def _update_window(self, df, data, window=None, resolution=None):
if df is None:
df = data
else:
df = df.append(data)
if resolution is not None:
try:
tz = str(df.index.tz)
except:
tz = None
df = tools.resample(df, resolution=resolution, tz=tz)
if window is None:
return df
return df[-window:]
# ---------------------------------------
# signal logging methods
# ---------------------------------------
def _add_signal_history(self, df, symbol):
""" Initilize signal history """
if symbol not in self.signals.keys() or len(self.signals[symbol]) == 0:
self.signals[symbol] = [nan]*len(df)
else:
self.signals[symbol].append(nan)
self.signals[symbol] = self.signals[symbol][-len(df):]
df.loc[-len(self.signals[symbol]):, 'signal'] = self.signals[symbol]
return df
def _log_signal(self, symbol, signal):
""" Log signal
:Parameters:
symbol : string
instruments symbol
signal : integer
signal identifier (1, 0, -1)
"""
self.signals[symbol][-1] = signal
class Reports():
"""Reports class initilizer
:Optional:
blotter : str
Log trades to MySQL server used by this Blotter (default is "auto detect")
port : int
HTTP port to use (default: 5000)
host : string
Host to bind the http process to (defaults 0.0.0.0)
password : string
Password for logging in to the web app (auto-generated by default). Use "" for no password.
"""
# ---------------------------------------
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
self.dbconn = None
self.dbcurr = None
self.host = args.host if args.host is not None else host
self.port = args.port if args.port is not None else port
self.blotter_name = args.blotter if args.blotter is not None else blotter
self.load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# ---------------------------------------
def load_blotter_args(self, name=None):
if name is not None:
self.blotter_name = name
# find specific name
if self.blotter_name is not None: # and self.blotter_name != 'auto-detect':
args_cache_file = tempfile.gettempdir(
) + "/" + self.blotter_name.lower() + ".ezq"
if not os.path.exists(args_cache_file):
print("[ERROR] Cannot connect to running Blotter [%s]" %
(self.blotter_name))
sys.exit(0)
# no name provided - connect to last running
else:
blotter_files = sorted(glob.glob(tempfile.gettempdir() + "/*.ezq"),
key=os.path.getmtime)
if len(blotter_files) == 0:
print("[ERROR] Cannot connect to running Blotter [%s]" %
(self.blotter_name))
sys.exit(0)
args_cache_file = blotter_files[-1]
args = pickle.load(open(args_cache_file, "rb"))
args['as_client'] = True
if args:
# connect to mysql
self.dbconn = pymysql.connect(host=str(args['dbhost']),
port=int(args['dbport']),
user=str(args['dbuser']),
passwd=str(args['dbpass']),
db=str(args['dbname']),
autocommit=True)
self.dbcurr = self.dbconn.cursor()
self.blotter_args = args
# ---------------------------------------
def send_static(self, path):
return send_from_directory('_webapp/', path)
# ---------------------------------------
def login(self, password):
if self._password == password:
resp = make_response('yes')
resp.set_cookie('password', password)
return resp
else:
return make_response("no")
# ---------------------------------------
def algos(self, json=True):
algos = pd.read_sql("SELECT DISTINCT algo FROM trades",
self.dbconn).to_dict(orient="records")
if json:
return jsonify(algos)
else:
return algos
# ---------------------------------------
def symbols(self, json=True):
symbols = pd.read_sql("SELECT * FROM symbols",
self.dbconn).to_dict(orient="records")
if json:
return jsonify(symbols)
else:
return symbols
# ---------------------------------------
def trades(self, start=None, end=None, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
trades_query = "SELECT * FROM trades WHERE exit_time IS NOT NULL"
trades_where = []
if isinstance(start, str):
trades_where.append("entry_time>='" + start + "'")
if isinstance(end, str):
trades_where.append("exit_time<='" + end + "'")
if algo_id is not None:
trades_where.append("algo='" + algo_id + "'")
if len(trades_where) > 0:
trades_query += " AND " + " AND ".join(trades_where)
trades = pd.read_sql(trades_query, self.dbconn)
trades['exit_time'].fillna(0, inplace=True)
trades['slippage'] = abs(trades['entry_price'] -
trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") &
(trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") &
(trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['exit_time', 'entry_time'],
ascending=[False, False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def positions(self, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
trades_query = "SELECT * FROM trades WHERE exit_time IS NULL"
if algo_id is not None:
trades_query += " AND algo='" + algo_id + "'"
trades = pd.read_sql(trades_query, self.dbconn)
last_query = "SELECT s.id, s.symbol, max(t.last) as last_price FROM ticks t LEFT JOIN symbols s ON (s.id=t.symbol_id) GROUP BY s.id"
last_prices = pd.read_sql(last_query, self.dbconn)
trades = trades.merge(last_prices, on=['symbol'])
trades['unrealized_pnl'] = np.where(
trades['direction'] == "SHORT",
trades['entry_price'] - trades['last_price'],
trades['last_price'] - trades['entry_price'])
trades['slippage'] = abs(trades['entry_price'] -
trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") &
(trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") &
(trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['entry_time'], ascending=[False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def trades_by_algo(self, algo_id=None, start=None, end=None):
trades = self.trades(start, end, algo_id=algo_id, json=False)
return jsonify(trades)
# ---------------------------------------
def bars(self, resolution, symbol, start=None, end=None, json=True):
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
bars = self.blotter.history(symbols=symbol,
start=start,
end=end,
resolution=resolution)
bars['datetime'] = bars.index
bars = bars.to_dict(orient="records")
if json:
return jsonify(bars)
else:
return bars
# ---------------------------------------
def index(self, start=None, end=None):
if not args.nopass:
if self._password != "" and self._password != request.cookies.get(
'password'):
return render_template('login.html')
return render_template('dashboard.html')
# ---------------------------------------
def run(self):
"""Starts the reporting module
Makes the dashboard web app available via localhost:port, and exposes
a REST API for trade information, open positions and market data.
"""
global app
# -----------------------------------
# assign view
app.add_url_rule('/', 'index', view_func=self.index)
app.add_url_rule('/<path:start>', 'index', view_func=self.index)
app.add_url_rule('/<start>/<path:end>', 'index', view_func=self.index)
app.add_url_rule('/algos', 'algos', view_func=self.algos)
app.add_url_rule('/symbols', 'symbols', view_func=self.symbols)
app.add_url_rule('/positions', 'positions', view_func=self.positions)
app.add_url_rule('/positions/<path:algo_id>',
'positions',
view_func=self.positions)
app.add_url_rule('/algo/<path:algo_id>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<path:start>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<start>/<path:end>',
'trades_by_algo',
view_func=self.trades_by_algo)
app.add_url_rule('/bars/<resolution>/<symbol>',
'bars',
view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<path:start>',
'bars',
view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<start>/<path:end>',
'bars',
view_func=self.bars)
app.add_url_rule('/trades', 'trades', view_func=self.trades)
app.add_url_rule('/trades/<path:start>',
'trades',
view_func=self.trades)
app.add_url_rule('/trades/<start>/<path:end>',
'trades',
view_func=self.trades)
app.add_url_rule('/login/<password>', 'login', view_func=self.login)
app.add_url_rule('/static/<path>',
'send_static',
view_func=self.send_static)
# let user know what the temp password is
if not args.nopass and self._password != "":
print(" * Web app password is:", self._password)
# notice
print(" * Running on http://" + str(self.host) + ":" + str(self.port) +
"/ (Press CTRL+C to quit)")
# -----------------------------------
# run flask app
app.run(
# debug = True,
host=str(self.host),
port=int(self.port))
class Reports():
"""Reports class initilizer
:Optional:
blotter : str
Log trades to MySQL server used by this Blotter (default is "auto detect")
port : int
HTTP port to use (default: 5000)
host : string
Host to bind the http process to (defaults 0.0.0.0)
password : string
Password for logging in to the web app (auto-generated by default). Use "" for no password.
"""
# ---------------------------------------
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {arg: val for arg, val in locals().items(
) if arg not in ('__class__', 'self', 'kwargs')}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(
host = str(self.blotter_args['dbhost']),
port = int(self.blotter_args['dbport']),
user = str(self.blotter_args['dbuser']),
passwd = str(self.blotter_args['dbpass']),
db = str(self.blotter_args['dbname']),
autocommit = True
)
self.dbcurr = self.dbconn.cursor()
# ---------------------------------------
def load_cli_args(self):
"""
Parse command line arguments and return only the non-default ones
:Retruns: dict
a dict of any non-default args passed on the command-line.
"""
parser = argparse.ArgumentParser(description='QTPyLib Reporting',
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument('--port', default=self.args["port"],
help='HTTP port to use', type=int)
parser.add_argument('--host', default=self.args["host"],
help='Host to bind the http process to')
parser.add_argument('--blotter',
help='Use this Blotter\'s MySQL server settings')
parser.add_argument('--nopass',
help='Skip password for web app (flag)',
action='store_true')
# only return non-default cmd line args
# (meaning only those actually given)
cmd_args, unknown = parser.parse_known_args()
args = {arg: val for arg, val in vars(cmd_args).items() if val != parser.get_default(arg)}
return args
# ---------------------------------------
def send_static(self, path):
return send_from_directory('_webapp/', path)
# ---------------------------------------
def login(self, password):
if self._password == password:
resp = make_response('yes')
resp.set_cookie('password', password)
return resp
else:
return make_response("no")
# ---------------------------------------
def algos(self, json=True):
algos = pd.read_sql("SELECT DISTINCT algo FROM trades",
self.dbconn).to_dict(orient="records")
if json:
return jsonify(algos)
else:
return algos
# ---------------------------------------
def symbols(self, json=True):
symbols = pd.read_sql("SELECT * FROM symbols", self.dbconn).to_dict(orient="records")
if json:
return jsonify(symbols)
else:
return symbols
# ---------------------------------------
def trades(self, start=None, end=None, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
trades_query = "SELECT * FROM trades WHERE exit_time IS NOT NULL"
trades_where = []
if isinstance(start, str):
trades_where.append("entry_time>='" + start + "'")
if isinstance(end, str):
trades_where.append("exit_time<='" + end + "'")
if algo_id is not None:
trades_where.append("algo='" + algo_id + "'")
if len(trades_where) > 0:
trades_query += " AND " + " AND ".join(trades_where)
trades = pd.read_sql(trades_query, self.dbconn)
trades['exit_time'].fillna(0, inplace=True)
trades['slippage'] = abs(trades['entry_price'] - trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") & (trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") & (trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['exit_time', 'entry_time'], ascending=[False, False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def positions(self, algo_id=None, json=True):
if algo_id is not None:
algo_id = algo_id.replace('/', '')
trades_query = "SELECT * FROM trades WHERE exit_time IS NULL"
if algo_id is not None:
trades_query += " AND algo='" + algo_id + "'"
trades = pd.read_sql(trades_query, self.dbconn)
last_query = "SELECT s.id, s.symbol, max(t.last) as last_price FROM ticks t LEFT JOIN symbols s ON (s.id=t.symbol_id) GROUP BY s.id"
last_prices = pd.read_sql(last_query, self.dbconn)
trades = trades.merge(last_prices, on=['symbol'])
trades['unrealized_pnl'] = np.where(
trades['direction']=="SHORT",
trades['entry_price']-trades['last_price'],
trades['last_price']-trades['entry_price'])
trades['slippage'] = abs(trades['entry_price'] - trades['market_price'])
trades['slippage'] = np.where(
((trades['direction'] == "LONG") & (trades['entry_price'] > trades['market_price'])) |
((trades['direction'] == "SHORT") & (trades['entry_price'] < trades['market_price'])),
-trades['slippage'], trades['slippage'])
trades = trades.sort_values(['entry_time'], ascending=[False])
trades = trades.to_dict(orient="records")
if json:
return jsonify(trades)
else:
return trades
# ---------------------------------------
def trades_by_algo(self, algo_id=None, start=None, end=None):
trades = self.trades(start, end, algo_id=algo_id, json=False)
return jsonify(trades)
# ---------------------------------------
def bars(self, resolution, symbol, start=None, end=None, json=True):
if start is not None:
start = start.replace('/', '')
if end is not None:
end = end.replace('/', '')
if start is None:
start = tools.backdate("7D", date=None, as_datetime=False)
bars = self.blotter.history(
symbols = symbol,
start = start,
end = end,
resolution = resolution
)
bars['datetime'] = bars.index
bars = bars.to_dict(orient="records")
if json:
return jsonify(bars)
else:
return bars
# ---------------------------------------
def index(self, start=None, end=None):
if not self.args['nopass']:
if self._password != "" and self._password != request.cookies.get('password'):
return render_template('login.html')
return render_template('dashboard.html')
# ---------------------------------------
def run(self):
"""Starts the reporting module
Makes the dashboard web app available via localhost:port, and exposes
a REST API for trade information, open positions and market data.
"""
global app
# -----------------------------------
# assign view
app.add_url_rule('/', 'index', view_func=self.index)
app.add_url_rule('/<path:start>', 'index', view_func=self.index)
app.add_url_rule('/<start>/<path:end>', 'index', view_func=self.index)
app.add_url_rule('/algos', 'algos', view_func=self.algos)
app.add_url_rule('/symbols', 'symbols', view_func=self.symbols)
app.add_url_rule('/positions', 'positions', view_func=self.positions)
app.add_url_rule('/positions/<path:algo_id>', 'positions', view_func=self.positions)
app.add_url_rule('/algo/<path:algo_id>', 'trades_by_algo', view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<path:start>',
'trades_by_algo', view_func=self.trades_by_algo)
app.add_url_rule('/algo/<algo_id>/<start>/<path:end>',
'trades_by_algo', view_func=self.trades_by_algo)
app.add_url_rule('/bars/<resolution>/<symbol>', 'bars', view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<path:start>', 'bars', view_func=self.bars)
app.add_url_rule('/bars/<resolution>/<symbol>/<start>/<path:end>',
'bars', view_func=self.bars)
app.add_url_rule('/trades', 'trades', view_func=self.trades)
app.add_url_rule('/trades/<path:start>', 'trades', view_func=self.trades)
app.add_url_rule('/trades/<start>/<path:end>', 'trades', view_func=self.trades)
app.add_url_rule('/login/<password>', 'login', view_func=self.login)
app.add_url_rule('/static/<path>', 'send_static', view_func=self.send_static)
# let user know what the temp password is
if not self.args['nopass'] and self._password != "":
print(" * Web app password is:", self._password)
# notice
# print(" * Running on http://"+ str(self.host) +":"+str(self.port)+"/ (Press CTRL+C to quit)")
# -----------------------------------
# run flask app
app.run(
debug=True,
host=str(self.host),
port=int(self.port)
)
class Algo(Broker):
"""Algo class initilizer (sub-class of Broker)
:Parameters:
instruments : list
List of IB contract tuples
resolution : str
Desired bar resolution (using pandas resolution: 1T, 1H, etc). Use K for tick bars.
tick_window : int
Length of tick lookback window to keep. Defaults to 1
bar_window : int
Length of bar lookback window to keep. Defaults to 100
timezone : str
Convert IB timestamps to this timezone (eg. US/Central). Defaults to UTC
preload : str
Preload history when starting algo (using pandas resolution: 1H, 1D, etc). Use K for tick bars.
blotter : str
Log trades to MySQL server used by this Blotter (default is "auto detect")
"""
__metaclass__ = ABCMeta
def __init__(self, instruments, resolution, \
tick_window=1, bar_window=100, timezone="UTC", preload=None, \
blotter=None, **kwargs):
self.name = str(self.__class__).split('.')[-1].split("'")[0]
# assign algo params
self.bars = pd.DataFrame()
self.ticks = pd.DataFrame()
self.quotes = {}
self.tick_count = 0
self.tick_bar_count = 0
self.bar_count = 0
self.bar_hash = 0
self.tick_window = tick_window if tick_window > 0 else 1
self.bar_window = bar_window if bar_window > 0 else 100
self.resolution = resolution.replace("MIN", "T")
self.timezone = timezone
self.preload = preload
self.backtest = args.backtest
self.backtest_start = args.start
self.backtest_end = args.end
# -----------------------------------
self.sms_numbers = [] if args.sms is None else args.sms
self.trade_log_dir = args.log
self.blotter_name = args.blotter if args.blotter is not None else blotter
self.record_output = args.output
# -----------------------------------
# load blotter settings && initilize Blotter
self.load_blotter_args(args.blotter)
self.blotter = Blotter(**self.blotter_args)
# -----------------------------------
# initiate broker/order manager
super().__init__(instruments, ibclient=int(args.ibclient), \
ibport=int(args.ibport), ibserver=str(args.ibserver))
# -----------------------------------
# signal collector
self.signals = {}
for sym in self.symbols:
self.signals[sym] = pd.DataFrame()
# -----------------------------------
# initilize output file
self.record_ts = None
if self.record_output:
self.datastore = tools.DataStore(args.output)
# -----------------------------------
# initiate strategy
self.on_start()
# ---------------------------------------
def run(self):
"""Starts the algo
Connects to the Blotter, processes market data and passes
tick data to the ``on_tick`` function and bar data to the
``on_bar`` methods.
"""
# -----------------------------------
# backtest mode?
if self.backtest:
if self.output is None:
print(
"ERROR: Must provide an output file for backtesting mode")
sys.exit(0)
if self.backtest_start is None:
print("ERROR: Must provide start date for backtesting mode")
sys.exit(0)
if self.backtest_end is None:
self.backtest_end = datetime.now().strftime(
'%Y-%m-%d %H:%M:%S.%f')
# backtest history
self.blotter.drip(symbols=self.symbols,
start=self.backtest_start,
end=self.backtest_end,
resolution=self.resolution,
tz=self.timezone,
quote_handler=self._quote_handler,
tick_handler=self._tick_handler,
bar_handler=self._bar_handler)
# -----------------------------------
# live data mode
else:
# preload history
if self.preload is not None:
try: # dbskip may be active
self.bars = self.blotter.history(
symbols=self.symbols,
start=tools.backdate(self.preload),
resolution=self.resolution,
tz=self.timezone)
except:
pass
# print(self.bars)
# add instruments to blotter in case they do not exist
self.blotter.register(self.instruments)
# listen for RT data
self.blotter.listen(symbols=self.symbols,
tz=self.timezone,
quote_handler=self._quote_handler,
tick_handler=self._tick_handler,
bar_handler=self._bar_handler)
# ---------------------------------------
@abstractmethod
def on_start(self):
"""
Invoked once when algo starts. Used for when the strategy
needs to initialize parameters upon starting.
"""
# raise NotImplementedError("Should implement on_start()")
pass
# ---------------------------------------
@abstractmethod
def on_quote(self, instrument):
"""
Invoked on every quote captured for the selected instrument.
This is where you'll write your strategy logic for quote events.
:Parameters:
symbol : string
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_quote()")
pass
# ---------------------------------------
@abstractmethod
def on_tick(self, instrument):
"""
Invoked on every tick captured for the selected instrument.
This is where you'll write your strategy logic for tick events.
:Parameters:
symbol : string
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_tick()")
pass
# ---------------------------------------
@abstractmethod
def on_bar(self, instrument):
"""
Invoked on every tick captured for the selected instrument.
This is where you'll write your strategy logic for tick events.
:Parameters:
instrument : object
`Instruments Object <#instrument-api>`_
"""
# raise NotImplementedError("Should implement on_bar()")
pass
# ---------------------------------------
@abstractmethod
def on_fill(self, instrument, order):
"""
Invoked on every order fill for the selected instrument.
This is where you'll write your strategy logic for fill events.
:Parameters:
instrument : object
`Instruments Object <#instrument-api>`_
order : object
Filled order data
"""
# raise NotImplementedError("Should implement on_fill()")
pass
# ---------------------------------------
def get_instrument(self, symbol):
"""
A string subclass that provides easy access to misc
symbol-related methods and information using shorthand.
Refer to the `Instruments API <#instrument-api>`_
for available methods and properties
Call from within your strategy:
``instrument = self.get_instrument("SYMBOL")``
:Parameters:
symbol : string
instrument symbol
"""
instrument = Instrument(self._getsymbol_(symbol))
instrument._set_parent(self)
return instrument
# ---------------------------------------
def get_history(self, symbols, start, end=None, resolution="1T", tz="UTC"):
"""Get historical market data.
Connects to Blotter and gets historical data from storage
:Parameters:
symbols : list
List of symbols to fetch history for
start : datetime / string
History time period start date (datetime or YYYY-MM-DD[ HH:MM[:SS]] string)
:Optional:
end : datetime / string
History time period end date (datetime or YYYY-MM-DD[ HH:MM[:SS]] string)
resolution : string
History resoluton (Pandas resample, defaults to 1T/1min)
tz : string
History timezone (defaults to UTC)
:Returns:
history : pd.DataFrame
Pandas DataFrame object with historical data for all symbols
"""
return self.blotter.history(symbols, start, end, resolution, tz)
# ---------------------------------------
# shortcuts to broker._create_order
# ---------------------------------------
def order(self, signal, symbol, quantity=0, **kwargs):
""" Send an order for the selected instrument
:Parameters:
direction : string
Order Type (BUY/SELL, EXIT/FLATTEN)
symbol : string
instrument symbol
quantity : int
Order quantiry
:Optional:
limit_price : float
In case of a LIMIT order, this is the LIMIT PRICE
expiry : int
Cancel this order if not filled after *n* seconds (default 60 seconds)
order_type : string
Type of order: Market (default), LIMIT (default when limit_price is passed), MODIFY (required passing or orderId)
orderId : int
If modifying an order, the order id of the modified order
target : float
target (exit) price
initial_stop : float
price to set hard stop
trail_stop_at : float
price at which to start trailing the stop
trail_stop_by : float
% of trailing stop distance from current price
ticksize : float
If using traling stop, pass the tick size for the instruments so you won't try to buy ES at 2200.128230 :)
fillorkill: bool
fill entire quantiry or none at all
iceberg: bool
is this an iceberg (hidden) order
"""
if signal.upper() == "EXIT" or signal.upper() == "FLATTEN":
position = self.get_positions(symbol)
if position['position'] == 0:
return
kwargs['symbol'] = symbol
kwargs['quantity'] = abs(position['position'])
kwargs['direction'] = "BUY" if position['position'] < 0 else "SELL"
# print("EXIT", kwargs)
try:
self.record(position=0)
except:
pass
if not self.backtest:
self._create_order(**kwargs)
else:
if quantity == 0:
return
kwargs['symbol'] = symbol
kwargs['quantity'] = abs(quantity)
kwargs['direction'] = signal.upper()
# print(signal.upper(), kwargs)
# record
try:
quantity = -quantity if kwargs[
'direction'] == "BUY" else quantity
self.record(position=quantity)
except:
pass
if not self.backtest:
self._create_order(**kwargs)
# ---------------------------------------
def cancel_order(self, orderId):
""" Cancels a un-filled order
Parameters:
orderId : int
Order ID
"""
self._cancel_order(orderId)
# ---------------------------------------
def record(self, *args, **kwargs):
"""Records data for later analysis.
Values will be logged to the file specified via
``--output [file]`` (along with bar data) as
csv/pickle/h5 file.
Call from within your strategy:
``self.record(key=value)``
:Parameters:
** kwargs : mixed
The names and values to record
"""
if self.record_output:
self.datastore.record(self.record_ts, *args, **kwargs)
# ---------------------------------------
def sms(self, text):
"""Sends an SMS message.
Relies on properly setting up an SMS provider (refer to the
SMS section of the documentation for more information about this)
Call from within your strategy:
``self.sms("message text")``
:Parameters:
text : string
The body of the SMS message to send
"""
logging.info("SMS: " + str(text))
sms.send_text(self.name + ': ' + str(text), self.sms_numbers)
# ---------------------------------------
def _caller(self, caller):
stack = [x[3] for x in inspect.stack()][1:-1]
return caller in stack
# ---------------------------------------
def _quote_handler(self, quote):
# self._cancel_expired_pending_orders()
self.quotes[quote['symbol']] = quote
self.on_quote(self.get_instrument(quote))
# ---------------------------------------
def _tick_handler(self, tick):
self._cancel_expired_pending_orders()
if "K" not in self.resolution:
self.ticks = self._update_window(self.ticks,
tick,
window=self.tick_window)
else:
self.ticks = self._update_window(self.ticks, tick)
bar = tools.resample(self.ticks, self.resolution)
if len(bar) > self.tick_bar_count > 0:
self.record_ts = tick.index[0]
self._bar_handler(bar)
periods = int("".join(
[s for s in self.resolution if s.isdigit()]))
self.ticks = self.ticks[-periods:]
self.tick_bar_count = len(bar)
# record tick bar
self.record(bar)
self.on_tick(self.get_instrument(tick))
# ---------------------------------------
def _bar_handler(self, bar):
is_tick_bar = False
handle_bar = True
if "K" in self.resolution:
if self._caller("_tick_handler"):
is_tick_bar = True
handle_bar = True
else:
is_tick_bar = True
handle_bar = False
if is_tick_bar:
# just add a bar (used by tick bar bandler)
self.bars = self._update_window(self.bars,
bar,
window=self.bar_window)
else:
# add the bar and resample to resolution
self.bars = self._update_window(self.bars,
bar,
window=self.bar_window,
resolution=self.resolution)
# new bar?
this_bar_hash = abs(hash(str(self.bars.index.values[-1]))) % (10**8)
newbar = (self.bar_hash != this_bar_hash)
self.bar_hash = this_bar_hash
if newbar & handle_bar:
self.record_ts = bar.index[0]
self.on_bar(self.get_instrument(bar))
if "K" not in self.resolution:
self.record(bar)
# ---------------------------------------
def _update_window(self, df, data, window=None, resolution=None):
if df is None:
df = data
else:
df = df.append(data)
if resolution is not None:
try:
tz = str(df.index.tz)
except:
tz = None
df = tools.resample(df, resolution=resolution, tz=tz)
if window is None:
return df
return df[-window:]
# ---------------------------------------
# signal logging methods
# ---------------------------------------
def _add_signal_history(self, df, symbol):
""" Initilize signal history """
if symbol not in self.signals.keys() or len(self.signals[symbol]) == 0:
self.signals[symbol] = [nan] * len(df)
else:
self.signals[symbol].append(nan)
self.signals[symbol] = self.signals[symbol][-len(df):]
df.loc[-len(self.signals[symbol]):, 'signal'] = self.signals[symbol]
return df
def _log_signal(self, symbol, signal):
""" Log signal
:Parameters:
symbol : string
instruments symbol
signal : integer
signal identifier (1, 0, -1)
"""
self.signals[symbol][-1] = signal
