def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {arg: val for arg, val in locals().items(
) if arg not in ('__class__', 'self', 'kwargs')}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(
host = str(self.blotter_args['dbhost']),
port = int(self.blotter_args['dbport']),
user = str(self.blotter_args['dbuser']),
passwd = str(self.blotter_args['dbpass']),
db = str(self.blotter_args['dbname']),
autocommit = True
)
self.dbcurr = self.dbconn.cursor()
def get_data_ib(instrument,
start,
resolution="1 min",
blotter=None,
output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(blotter,
logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']),
host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES",
resolution=resolution,
lookback=tools.ib_duration_str(start),
rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs):
# return
self._password = password if password is not None else hashlib.sha1(
str(datetime.datetime.now()
).encode()).hexdigest()[:6]
# initilize class logger
self.log = logging.getLogger(__name__)
# override args with any (non-default) command-line args
self.args = {arg: val for arg, val in locals().items() if arg not in ('__class__', 'self', 'kwargs')}
self.args.update(kwargs)
self.args.update(self.load_cli_args())
self.dbconn = None
self.dbcurr = None
self.host = self.args['host'] if self.args['host'] is not None else host
self.port = self.args['port'] if self.args['port'] is not None else port
# blotter / db connection
self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter
self.blotter_args = load_blotter_args(self.blotter_name)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
self.dbconn = pymysql.connect(
host = str(self.blotter_args['dbhost']),
port = int(self.blotter_args['dbport']),
user = str(self.blotter_args['dbuser']),
passwd = str(self.blotter_args['dbpass']),
db = str(self.blotter_args['dbname']),
autocommit = True
)
self.dbcurr = self.dbconn.cursor()
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(
blotter, logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']), host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"):
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# initilize class logger
self.log_broker = logging.getLogger(__name__)
# default params (overrided in algo)
self.timezone = "UTC"
self.last_price = {}
self.tick_window = 1000
self.bar_window = 100
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
else:
instrument = tools.create_ib_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except Exception as e:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
self.instrument_combos = {}
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(
by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={}
)
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = load_blotter_args(
self.blotter_name, logger=self.log_broker)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
if not self.blotter_args['dbskip']:
self.dbconn = pymysql.connect(
host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True
)
self.dbcurr = self.dbconn.cursor()
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
def store_data(df, blotter=None, kind="BAR"):
"""
Store QTPyLib-compatible csv files in Blotter's MySQL.
TWS/GW data are required for determining futures/options expiration
:Parameters:
df : dict
Tick/Bar data
:Optional:
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
kind : str
Is this ``BAR`` or ``TICK`` data
"""
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# load blotter settings
blotter_args = load_blotter_args(
blotter, logger=logging.getLogger(__name__))
# blotter not running
if blotter_args is None:
raise Exception("Cannot connect to running Blotter.")
# cannot continue
if blotter_args['dbskip']:
raise Exception("Cannot continue. Blotter running with --dbskip")
# connect to mysql using blotter's settings
dbconn = pymysql.connect(
client_flag=MULTI_STATEMENTS,
host=str(blotter_args['dbhost']),
port=int(blotter_args['dbport']),
user=str(blotter_args['dbuser']),
passwd=str(blotter_args['dbpass']),
db=str(blotter_args['dbname']),
autocommit=True
)
dbcurr = dbconn.cursor()
# loop through symbols and save in db
for symbol in list(df['symbol'].unique()):
data = df[df['symbol'] == symbol]
symbol_id = get_symbol_id(symbol, dbconn, dbcurr)
# prepare columns for insert
data.loc[:, 'timestamp'] = data.index.strftime('%Y-%m-%d %H:%M:%S')
data.loc[:, 'symbol_id'] = symbol_id
# insert row by row to handle greeks
data = data.to_dict(orient="records")
if kind == "BAR":
for _, row in enumerate(data):
mysql_insert_bar(row, symbol_id, dbcurr)
else:
for _, row in enumerate(data):
mysql_insert_tick(row, symbol_id, dbcurr)
try:
dbconn.commit()
except Exception as e:
return False
return True
def store_data(df, blotter=None, kind="BAR"):
"""
Store QTPyLib-compatible csv files in Blotter's MySQL.
TWS/GW data are required for determining futures/options expiration
:Parameters:
df : dict
Tick/Bar data
:Optional:
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
kind : str
Is this ``BAR`` or ``TICK`` data
"""
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# load blotter settings
blotter_args = load_blotter_args(blotter, logger=logging.getLogger(__name__))
# blotter not running
if blotter_args is None:
raise Exception("Cannot connect to running Blotter.")
return False
# cannot continue
if blotter_args['dbskip']:
raise Exception("Cannot continue. Blotter running with --dbskip")
return False
# connect to mysql using blotter's settings
dbconn = pymysql.connect(
host = str(blotter_args['dbhost']),
port = int(blotter_args['dbport']),
user = str(blotter_args['dbuser']),
passwd = str(blotter_args['dbpass']),
db = str(blotter_args['dbname']),
autocommit = True
)
dbcurr = dbconn.cursor()
# loop through symbols and save in db
for symbol in list(df['symbol'].unique()):
data = df[df['symbol'] == symbol]
symbol_id = get_symbol_id(symbol, dbconn, dbcurr)
# prepare columns for insert
data.loc[:, 'timestamp'] = data.index.strftime('%Y-%m-%d %H:%M:%S')
data.loc[:, 'symbol_id'] = symbol_id
# insert row by row to handle greeks
data = data.to_dict(orient="records")
if kind == "BAR":
for _, row in enumerate(data):
mysql_insert_bar(row, symbol_id, dbcurr)
else:
for _, row in enumerate(data):
mysql_insert_tick(row, symbol_id, dbcurr)
try:
dbconn.commit()
except:
return False
return True
