def get_data_ib(instrument,
start,
resolution="1 min",
blotter=None,
output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(blotter,
logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']),
host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES",
resolution=resolution,
lookback=tools.ib_duration_str(start),
rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None):
"""
Downloads historical data from Interactive Brokers
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
start : str
Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS])
:Optional:
resolution : str
1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day
blotter : str
Store MySQL server used by this Blotter (default is "auto detect")
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _ib_history_downloaded
# load blotter settings
blotter_args = load_blotter_args(
blotter, logger=logging.getLogger(__name__))
# create contract string (no need for connection)
ibConn = ezIBpy()
ibConn.ibCallback = ibCallback
if not ibConn.connected:
ibConn.connect(clientId=0,
port=int(blotter_args['ibport']), host=str(blotter_args['ibserver']))
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
contract_string = ibConn.contractString(instrument)
contract = ibConn.createContract(instrument)
ibConn.requestHistoricalData(contracts=[contract],
data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False)
while not _ib_history_downloaded:
time.sleep(.1)
ibConn.disconnect()
data = ibConn.historicalData[contract_string]
data['datetime'] = data.index
return prepare_data(instrument, data, output_path=output_path)
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"):
# detect running strategy
self.strategy = str(self.__class__).split('.')[-1].split("'")[0]
# initilize class logger
self.log_broker = logging.getLogger(__name__)
# default params (overrided in algo)
self.timezone = "UTC"
self.last_price = {}
self.tick_window = 1000
self.bar_window = 100
# -----------------------------------
# connect to IB
self.ibclient = int(ibclient)
self.ibport = int(ibport)
self.ibserver = str(ibserver)
self.ibConn = ezibpy.ezIBpy()
self.ibConn.ibCallback = self.ibCallback
self.ibConnect()
# -----------------------------------
# create contracts
instrument_tuples_dict = {}
for instrument in instruments:
try:
if isinstance(instrument, ezibpy.utils.Contract):
instrument = self.ibConn.contract_to_tuple(instrument)
else:
instrument = tools.create_ib_tuple(instrument)
contractString = self.ibConn.contractString(instrument)
instrument_tuples_dict[contractString] = instrument
self.ibConn.createContract(instrument)
except Exception as e:
pass
self.instruments = instrument_tuples_dict
self.symbols = list(self.instruments.keys())
self.instrument_combos = {}
# -----------------------------------
# track orders & trades
self.active_trades = {}
self.trades = []
# shortcut
self.account = self.ibConn.account
# use: self.orders.pending...
self.orders = tools.make_object(
by_tickerid=self.ibConn.orders,
by_symbol=self.ibConn.symbol_orders,
pending_ttls={},
pending={},
filled={},
active={},
history={},
nextId=1,
recent={}
)
# -----------------------------------
self.dbcurr = None
self.dbconn = None
# -----------------------------------
# assign default vals if not propogated from algo
if not hasattr(self, 'backtest'):
self.backtest = False
if not hasattr(self, 'sms_numbers'):
self.sms_numbers = []
if not hasattr(self, 'trade_log_dir'):
self.trade_log_dir = None
if not hasattr(self, 'blotter_name'):
self.blotter_name = None
# -----------------------------------
# load blotter settings
self.blotter_args = load_blotter_args(
self.blotter_name, logger=self.log_broker)
self.blotter = Blotter(**self.blotter_args)
# connect to mysql using blotter's settings
if not self.blotter_args['dbskip']:
self.dbconn = pymysql.connect(
host=str(self.blotter_args['dbhost']),
port=int(self.blotter_args['dbport']),
user=str(self.blotter_args['dbuser']),
passwd=str(self.blotter_args['dbpass']),
db=str(self.blotter_args['dbname']),
autocommit=True
)
self.dbcurr = self.dbconn.cursor()
# -----------------------------------
# do stuff on exit
atexit.register(self._on_exit)
def prepare_data(instrument, data, output_path=None,
index=None, colsmap=None, kind="BAR", resample="1T"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to where the resulting CSV should be saved (optional)
index : pd.Series
Pandas Series that will be used for df's index (optioanl)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib
(default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
resample : str
Pandas resolution (defaults to 1min/1T)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _TICKS_COLSMAP, _BARS_COLSMAP
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set([
'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# lower case columns
df.columns = map(str.lower, df.columns)
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + \
['symbol', 'symbol_group', 'asset_class', 'expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# resample
if resample and kind == "BAR":
df = tools.resample(df, resolution=resample, tz="UTC")
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
index : pd.Series
Pandas Series that will be used for df's index (default is to use df.index)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _bars_colsmap, _ticks_colsmap
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set(['datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _ticks_colsmap if kind == "TICK" else _bars_colsmap
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + ['symbol','symbol_group','asset_class','expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df