def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None): """ Downloads historical data from Interactive Brokers :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) start : str Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS]) :Optional: resolution : str 1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day blotter : str Store MySQL server used by this Blotter (default is "auto detect") output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _ib_history_downloaded # load blotter settings blotter_args = load_blotter_args(blotter, logger=logging.getLogger(__name__)) # create contract string (no need for connection) ibConn = ezIBpy() ibConn.ibCallback = ibCallback if not ibConn.connected: ibConn.connect(clientId=0, port=int(blotter_args['ibport']), host=str(blotter_args['ibserver'])) # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) contract_string = ibConn.contractString(instrument) contract = ibConn.createContract(instrument) ibConn.requestHistoricalData(contracts=[contract], data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False) while not _ib_history_downloaded: time.sleep(.1) ibConn.disconnect() data = ibConn.historicalData[contract_string] data['datetime'] = data.index return prepare_data(instrument, data, output_path=output_path)
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None): """ Downloads historical data from Interactive Brokers :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) start : str Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS]) :Optional: resolution : str 1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day blotter : str Store MySQL server used by this Blotter (default is "auto detect") output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _ib_history_downloaded # load blotter settings blotter_args = load_blotter_args( blotter, logger=logging.getLogger(__name__)) # create contract string (no need for connection) ibConn = ezIBpy() ibConn.ibCallback = ibCallback if not ibConn.connected: ibConn.connect(clientId=0, port=int(blotter_args['ibport']), host=str(blotter_args['ibserver'])) # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) contract_string = ibConn.contractString(instrument) contract = ibConn.createContract(instrument) ibConn.requestHistoricalData(contracts=[contract], data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False) while not _ib_history_downloaded: time.sleep(.1) ibConn.disconnect() data = ibConn.historicalData[contract_string] data['datetime'] = data.index return prepare_data(instrument, data, output_path=output_path)
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"): # detect running strategy self.strategy = str(self.__class__).split('.')[-1].split("'")[0] # initilize class logger self.log_broker = logging.getLogger(__name__) # default params (overrided in algo) self.timezone = "UTC" self.last_price = {} self.tick_window = 1000 self.bar_window = 100 # ----------------------------------- # connect to IB self.ibclient = int(ibclient) self.ibport = int(ibport) self.ibserver = str(ibserver) self.ibConn = ezibpy.ezIBpy() self.ibConn.ibCallback = self.ibCallback self.ibConnect() # ----------------------------------- # create contracts instrument_tuples_dict = {} for instrument in instruments: try: if isinstance(instrument, ezibpy.utils.Contract): instrument = self.ibConn.contract_to_tuple(instrument) else: instrument = tools.create_ib_tuple(instrument) contractString = self.ibConn.contractString(instrument) instrument_tuples_dict[contractString] = instrument self.ibConn.createContract(instrument) except Exception as e: pass self.instruments = instrument_tuples_dict self.symbols = list(self.instruments.keys()) self.instrument_combos = {} # ----------------------------------- # track orders & trades self.active_trades = {} self.trades = [] # shortcut self.account = self.ibConn.account # use: self.orders.pending... self.orders = tools.make_object( by_tickerid=self.ibConn.orders, by_symbol=self.ibConn.symbol_orders, pending_ttls={}, pending={}, filled={}, active={}, history={}, nextId=1, recent={} ) # ----------------------------------- self.dbcurr = None self.dbconn = None # ----------------------------------- # assign default vals if not propogated from algo if not hasattr(self, 'backtest'): self.backtest = False if not hasattr(self, 'sms_numbers'): self.sms_numbers = [] if not hasattr(self, 'trade_log_dir'): self.trade_log_dir = None if not hasattr(self, 'blotter_name'): self.blotter_name = None # ----------------------------------- # load blotter settings self.blotter_args = load_blotter_args( self.blotter_name, logger=self.log_broker) self.blotter = Blotter(**self.blotter_args) # connect to mysql using blotter's settings if not self.blotter_args['dbskip']: self.dbconn = pymysql.connect( host=str(self.blotter_args['dbhost']), port=int(self.blotter_args['dbport']), user=str(self.blotter_args['dbuser']), passwd=str(self.blotter_args['dbpass']), db=str(self.blotter_args['dbname']), autocommit=True ) self.dbcurr = self.dbconn.cursor() # ----------------------------------- # do stuff on exit atexit.register(self._on_exit)
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR", resample="1T"): """ Converts given DataFrame to a QTPyLib-compatible format and timezone :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) data : pd.DataFrame Pandas DataDrame with that instrument's market data output_path : str Path to where the resulting CSV should be saved (optional) index : pd.Series Pandas Series that will be used for df's index (optioanl) colsmap : dict Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's) kind : str Is this ``BAR`` or ``TICK`` data resample : str Pandas resolution (defaults to 1min/1T) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _TICKS_COLSMAP, _BARS_COLSMAP # work on copy df = data.copy() # ezibpy's csv? if set(df.columns) == set([ 'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']): df.rename(columns={ 'datetime': 'datetime', 'O': 'open', 'H': 'high', 'L': 'low', 'C': 'close', 'OI': 'volume', }, inplace=True) df.index = pd.to_datetime(df['datetime']) df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC") index = None # lower case columns df.columns = map(str.lower, df.columns) # set index if index is None: index = df.index # set defaults columns if not isinstance(colsmap, dict): colsmap = {} _colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP for el in _colsmap: if el not in colsmap: colsmap[el] = _colsmap[el] # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) # create contract string (no need for connection) ibConn = ezIBpy() contract_string = ibConn.contractString(instrument) asset_class = tools.gen_asset_class(contract_string) symbol_group = tools.gen_symbol_group(contract_string) # add symbol data df.loc[:, 'symbol'] = contract_string df.loc[:, 'symbol_group'] = symbol_group df.loc[:, 'asset_class'] = asset_class # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # rename columns to map df.rename(columns=colsmap, inplace=True) # force option columns on options if asset_class == "OPT": df = tools.force_options_columns(df) # remove all other columns known_cols = list(colsmap.values()) + \ ['symbol', 'symbol_group', 'asset_class', 'expiry'] for col in df.columns: if col not in known_cols: df.drop(col, axis=1, inplace=True) # set UTC index df.index = pd.to_datetime(index) df = tools.set_timezone(df, "UTC") df.index.rename("datetime", inplace=True) # resample if resample and kind == "BAR": df = tools.resample(df, resolution=resample, tz="UTC") # add expiry df.loc[:, 'expiry'] = np.nan if asset_class in ("FUT", "OPT", "FOP"): df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string) # save csv if output_path is not None: output_path = output_path[ :-1] if output_path.endswith('/') else output_path df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind)) # return df return df
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR"): """ Converts given DataFrame to a QTPyLib-compatible format and timezone :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) data : pd.DataFrame Pandas DataDrame with that instrument's market data output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) index : pd.Series Pandas Series that will be used for df's index (default is to use df.index) colsmap : dict Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's) kind : str Is this ``BAR`` or ``TICK`` data :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _bars_colsmap, _ticks_colsmap # work on copy df = data.copy() # ezibpy's csv? if set(df.columns) == set(['datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']): df.rename(columns={ 'datetime': 'datetime', 'O': 'open', 'H': 'high', 'L': 'low', 'C': 'close', 'OI': 'volume', }, inplace=True) df.index = pd.to_datetime(df['datetime']) df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC") index = None # set index if index is None: index = df.index # set defaults columns if not isinstance(colsmap, dict): colsmap = {} _colsmap = _ticks_colsmap if kind == "TICK" else _bars_colsmap for el in _colsmap: if el not in colsmap: colsmap[el] = _colsmap[el] # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) # create contract string (no need for connection) ibConn = ezIBpy() contract_string = ibConn.contractString(instrument) asset_class = tools.gen_asset_class(contract_string) symbol_group = tools.gen_symbol_group(contract_string) # add symbol data df.loc[:, 'symbol'] = contract_string df.loc[:, 'symbol_group'] = symbol_group df.loc[:, 'asset_class'] = asset_class # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # rename columns to map df.rename(columns=colsmap, inplace=True) # force option columns on options if asset_class == "OPT": df = tools.force_options_columns(df) # remove all other columns known_cols = list(colsmap.values()) + ['symbol','symbol_group','asset_class','expiry'] for col in df.columns: if col not in known_cols: df.drop(col, axis=1, inplace=True) # set UTC index df.index = pd.to_datetime(index) df = tools.set_timezone(df, "UTC") df.index.rename("datetime", inplace=True) # add expiry df.loc[:, 'expiry'] = np.nan if asset_class in ("FUT", "OPT", "FOP"): df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string) # save csv if output_path is not None: output_path = output_path[ :-1] if output_path.endswith('/') else output_path df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind)) # return df return df