def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs): # return self._password = password if password is not None else hashlib.sha1( str(datetime.datetime.now()).encode()).hexdigest()[:6] # initilize class logger self.log = logging.getLogger(__name__) # override args with any (non-default) command-line args self.args = {arg: val for arg, val in locals().items( ) if arg not in ('__class__', 'self', 'kwargs')} self.args.update(kwargs) self.args.update(self.load_cli_args()) self.dbconn = None self.dbcurr = None self.host = self.args['host'] if self.args['host'] is not None else host self.port = self.args['port'] if self.args['port'] is not None else port # blotter / db connection self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter self.blotter_args = load_blotter_args(self.blotter_name) self.blotter = Blotter(**self.blotter_args) # connect to mysql using blotter's settings self.dbconn = pymysql.connect( host = str(self.blotter_args['dbhost']), port = int(self.blotter_args['dbport']), user = str(self.blotter_args['dbuser']), passwd = str(self.blotter_args['dbpass']), db = str(self.blotter_args['dbname']), autocommit = True ) self.dbcurr = self.dbconn.cursor()
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None): """ Downloads historical data from Interactive Brokers :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) start : str Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS]) :Optional: resolution : str 1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day blotter : str Store MySQL server used by this Blotter (default is "auto detect") output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _ib_history_downloaded # load blotter settings blotter_args = load_blotter_args(blotter, logger=logging.getLogger(__name__)) # create contract string (no need for connection) ibConn = ezIBpy() ibConn.ibCallback = ibCallback if not ibConn.connected: ibConn.connect(clientId=0, port=int(blotter_args['ibport']), host=str(blotter_args['ibserver'])) # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) contract_string = ibConn.contractString(instrument) contract = ibConn.createContract(instrument) ibConn.requestHistoricalData(contracts=[contract], data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False) while not _ib_history_downloaded: time.sleep(.1) ibConn.disconnect() data = ibConn.historicalData[contract_string] data['datetime'] = data.index return prepare_data(instrument, data, output_path=output_path)
def __init__(self, blotter=None, port=5000, host="0.0.0.0", password=None, nopass=False, **kwargs): # return self._password = password if password is not None else hashlib.sha1( str(datetime.datetime.now() ).encode()).hexdigest()[:6] # initilize class logger self.log = logging.getLogger(__name__) # override args with any (non-default) command-line args self.args = {arg: val for arg, val in locals().items() if arg not in ('__class__', 'self', 'kwargs')} self.args.update(kwargs) self.args.update(self.load_cli_args()) self.dbconn = None self.dbcurr = None self.host = self.args['host'] if self.args['host'] is not None else host self.port = self.args['port'] if self.args['port'] is not None else port # blotter / db connection self.blotter_name = self.args['blotter'] if self.args['blotter'] is not None else blotter self.blotter_args = load_blotter_args(self.blotter_name) self.blotter = Blotter(**self.blotter_args) # connect to mysql using blotter's settings self.dbconn = pymysql.connect( host = str(self.blotter_args['dbhost']), port = int(self.blotter_args['dbport']), user = str(self.blotter_args['dbuser']), passwd = str(self.blotter_args['dbpass']), db = str(self.blotter_args['dbname']), autocommit = True ) self.dbcurr = self.dbconn.cursor()
def get_data_ib(instrument, start, resolution="1 min", blotter=None, output_path=None): """ Downloads historical data from Interactive Brokers :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) start : str Backtest start date (YYYY-MM-DD [HH:MM:SS[.MS]) :Optional: resolution : str 1 sec, 5 secs, 15 secs, 30 secs, 1 min (default), 2 mins, 3 mins, 5 mins, 15 mins, 30 mins, 1 hour, 1 day blotter : str Store MySQL server used by this Blotter (default is "auto detect") output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _ib_history_downloaded # load blotter settings blotter_args = load_blotter_args( blotter, logger=logging.getLogger(__name__)) # create contract string (no need for connection) ibConn = ezIBpy() ibConn.ibCallback = ibCallback if not ibConn.connected: ibConn.connect(clientId=0, port=int(blotter_args['ibport']), host=str(blotter_args['ibserver'])) # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) contract_string = ibConn.contractString(instrument) contract = ibConn.createContract(instrument) ibConn.requestHistoricalData(contracts=[contract], data="TRADES", resolution=resolution, lookback=tools.ib_duration_str(start), rth=False) while not _ib_history_downloaded: time.sleep(.1) ibConn.disconnect() data = ibConn.historicalData[contract_string] data['datetime'] = data.index return prepare_data(instrument, data, output_path=output_path)
def __init__(self, instruments, ibclient=998, ibport=4001, ibserver="localhost"): # detect running strategy self.strategy = str(self.__class__).split('.')[-1].split("'")[0] # initilize class logger self.log_broker = logging.getLogger(__name__) # default params (overrided in algo) self.timezone = "UTC" self.last_price = {} self.tick_window = 1000 self.bar_window = 100 # ----------------------------------- # connect to IB self.ibclient = int(ibclient) self.ibport = int(ibport) self.ibserver = str(ibserver) self.ibConn = ezibpy.ezIBpy() self.ibConn.ibCallback = self.ibCallback self.ibConnect() # ----------------------------------- # create contracts instrument_tuples_dict = {} for instrument in instruments: try: if isinstance(instrument, ezibpy.utils.Contract): instrument = self.ibConn.contract_to_tuple(instrument) else: instrument = tools.create_ib_tuple(instrument) contractString = self.ibConn.contractString(instrument) instrument_tuples_dict[contractString] = instrument self.ibConn.createContract(instrument) except Exception as e: pass self.instruments = instrument_tuples_dict self.symbols = list(self.instruments.keys()) self.instrument_combos = {} # ----------------------------------- # track orders & trades self.active_trades = {} self.trades = [] # shortcut self.account = self.ibConn.account # use: self.orders.pending... self.orders = tools.make_object( by_tickerid=self.ibConn.orders, by_symbol=self.ibConn.symbol_orders, pending_ttls={}, pending={}, filled={}, active={}, history={}, nextId=1, recent={} ) # ----------------------------------- self.dbcurr = None self.dbconn = None # ----------------------------------- # assign default vals if not propogated from algo if not hasattr(self, 'backtest'): self.backtest = False if not hasattr(self, 'sms_numbers'): self.sms_numbers = [] if not hasattr(self, 'trade_log_dir'): self.trade_log_dir = None if not hasattr(self, 'blotter_name'): self.blotter_name = None # ----------------------------------- # load blotter settings self.blotter_args = load_blotter_args( self.blotter_name, logger=self.log_broker) self.blotter = Blotter(**self.blotter_args) # connect to mysql using blotter's settings if not self.blotter_args['dbskip']: self.dbconn = pymysql.connect( host=str(self.blotter_args['dbhost']), port=int(self.blotter_args['dbport']), user=str(self.blotter_args['dbuser']), passwd=str(self.blotter_args['dbpass']), db=str(self.blotter_args['dbname']), autocommit=True ) self.dbcurr = self.dbconn.cursor() # ----------------------------------- # do stuff on exit atexit.register(self._on_exit)
def store_data(df, blotter=None, kind="BAR"): """ Store QTPyLib-compatible csv files in Blotter's MySQL. TWS/GW data are required for determining futures/options expiration :Parameters: df : dict Tick/Bar data :Optional: blotter : str Store MySQL server used by this Blotter (default is "auto detect") kind : str Is this ``BAR`` or ``TICK`` data """ # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # load blotter settings blotter_args = load_blotter_args( blotter, logger=logging.getLogger(__name__)) # blotter not running if blotter_args is None: raise Exception("Cannot connect to running Blotter.") # cannot continue if blotter_args['dbskip']: raise Exception("Cannot continue. Blotter running with --dbskip") # connect to mysql using blotter's settings dbconn = pymysql.connect( client_flag=MULTI_STATEMENTS, host=str(blotter_args['dbhost']), port=int(blotter_args['dbport']), user=str(blotter_args['dbuser']), passwd=str(blotter_args['dbpass']), db=str(blotter_args['dbname']), autocommit=True ) dbcurr = dbconn.cursor() # loop through symbols and save in db for symbol in list(df['symbol'].unique()): data = df[df['symbol'] == symbol] symbol_id = get_symbol_id(symbol, dbconn, dbcurr) # prepare columns for insert data.loc[:, 'timestamp'] = data.index.strftime('%Y-%m-%d %H:%M:%S') data.loc[:, 'symbol_id'] = symbol_id # insert row by row to handle greeks data = data.to_dict(orient="records") if kind == "BAR": for _, row in enumerate(data): mysql_insert_bar(row, symbol_id, dbcurr) else: for _, row in enumerate(data): mysql_insert_tick(row, symbol_id, dbcurr) try: dbconn.commit() except Exception as e: return False return True
def store_data(df, blotter=None, kind="BAR"): """ Store QTPyLib-compatible csv files in Blotter's MySQL. TWS/GW data are required for determining futures/options expiration :Parameters: df : dict Tick/Bar data :Optional: blotter : str Store MySQL server used by this Blotter (default is "auto detect") kind : str Is this ``BAR`` or ``TICK`` data """ # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # load blotter settings blotter_args = load_blotter_args(blotter, logger=logging.getLogger(__name__)) # blotter not running if blotter_args is None: raise Exception("Cannot connect to running Blotter.") return False # cannot continue if blotter_args['dbskip']: raise Exception("Cannot continue. Blotter running with --dbskip") return False # connect to mysql using blotter's settings dbconn = pymysql.connect( host = str(blotter_args['dbhost']), port = int(blotter_args['dbport']), user = str(blotter_args['dbuser']), passwd = str(blotter_args['dbpass']), db = str(blotter_args['dbname']), autocommit = True ) dbcurr = dbconn.cursor() # loop through symbols and save in db for symbol in list(df['symbol'].unique()): data = df[df['symbol'] == symbol] symbol_id = get_symbol_id(symbol, dbconn, dbcurr) # prepare columns for insert data.loc[:, 'timestamp'] = data.index.strftime('%Y-%m-%d %H:%M:%S') data.loc[:, 'symbol_id'] = symbol_id # insert row by row to handle greeks data = data.to_dict(orient="records") if kind == "BAR": for _, row in enumerate(data): mysql_insert_bar(row, symbol_id, dbcurr) else: for _, row in enumerate(data): mysql_insert_tick(row, symbol_id, dbcurr) try: dbconn.commit() except: return False return True