def _update_window(self, df, data, window=None, resolution=None): if df is None: df = data else: df = df.append(data) # resample if resolution is not None: try: tz = str(df.index.tz) except: tz = None df = tools.resample(df, resolution=resolution, tz=tz) # remove duplicates rows df.loc[:, '_idx_'] = df.index df.drop_duplicates( subset=['_idx_', 'symbol', 'symbol_group', 'asset_class'], keep='last', inplace=True) df.drop('_idx_', axis=1, inplace=True) # return if window is None: return df # return df[-window:] return self._get_window_per_symbol(df, window)
def _tick_handler(self, tick, stale_tick=False): self._cancel_expired_pending_orders() # initial value if self.record_ts is None: self.record_ts = tick.index[0] if self.resolution[-1] not in ("S", "K", "V"): self.ticks = self._update_window(self.ticks, tick, window=self.tick_window) else: self.ticks = self._update_window(self.ticks, tick) bars = tools.resample(self.ticks, self.resolution) if len(bars.index) > self.tick_bar_count > 0 or stale_tick: self.record_ts = tick.index[0] self._bar_handler(bars) periods = int("".join([s for s in self.resolution if s.isdigit()])) self.ticks = self.ticks[-periods:] self.tick_bar_count = len(bars) # record tick bars self.record(bars[-1:]) if not stale_tick: self.on_tick(self.get_instrument(tick))
def _tick_handler(self, tick, stale_tick=False): self._cancel_expired_pending_orders() # tick symbol symbol = tick['symbol'].values if len(symbol) == 0: return symbol = symbol[0] self.last_price[symbol] = float(tick['last'].values[0]) # work on copy self_ticks = self.ticks.copy() # initial value if self.record_ts is None: self.record_ts = tick.index[0] if self.resolution[-1] not in ("S", "K", "V"): if self.threads == 0: self.ticks = self._update_window( self.ticks, tick, window=self.tick_window) else: self_ticks = self._update_window( self_ticks, tick, window=self.tick_window) self.ticks = self._thread_safe_merge( symbol, self.ticks, self_ticks) # assign back else: self.ticks = self._update_window(self.ticks, tick) # bars = tools.resample(self.ticks, self.resolution) bars = tools.resample( self.ticks, self.resolution, tz=self.timezone) if len(bars.index) > self.tick_bar_count > 0 or stale_tick: self.record_ts = tick.index[0] self._base_bar_handler(bars[bars['symbol'] == symbol][-1:]) window = int( "".join([s for s in self.resolution if s.isdigit()])) if self.threads == 0: self.ticks = self._get_window_per_symbol( self.ticks, window) else: self_ticks = self._get_window_per_symbol( self_ticks, window) self.ticks = self._thread_safe_merge( symbol, self.ticks, self_ticks) # assign back self.tick_bar_count = len(bars.index) # record non time-based bars self.record(bars[-1:]) if not stale_tick: if self.ticks[(self.ticks['symbol'] == symbol) | ( self.ticks['symbol_group'] == symbol)].empty: return tick_instrument = self.get_instrument(tick) if tick_instrument: self.on_tick(tick_instrument)
def _update_window(self, df, data, window=None, resolution=None): if df is None: df = data else: df = df.append(data) if resolution is not None: try: tz = str(df.index.tz) except: tz = None df = tools.resample(df, resolution=resolution, tz=tz) if window is None: return df return df[-window:]
def _tick_handler(self, tick): self._cancel_expired_pending_orders() if "K" not in self.resolution and "V" not in self.resolution: self.ticks = self._update_window(self.ticks, tick, window=self.tick_window) else: self.ticks = self._update_window(self.ticks, tick) bar = tools.resample(self.ticks, self.resolution) if len(bar.index) > self.tick_bar_count > 0: self.record_ts = tick.index[0] self._bar_handler(bar) periods = int("".join([s for s in self.resolution if s.isdigit()])) self.ticks = self.ticks[-periods:] self.tick_bar_count = len(bar) # record tick bar self.record(bar) self.on_tick(self.get_instrument(tick))
def _tick_handler(self, tick, stale_tick=False): self._cancel_expired_pending_orders() # tick symbol symbol = tick['symbol'].values[0] self_ticks = self.ticks.copy() # work on copy # initial value if self.record_ts is None: self.record_ts = tick.index[0] if self.resolution[-1] not in ("S", "K", "V"): if self.threads == 0: self.ticks = self._update_window(self.ticks, tick, window=self.tick_window) else: self_ticks = self._update_window(self_ticks, tick, window=self.tick_window) self.ticks = self._thread_safe_merge(symbol, self.ticks, self_ticks) # assign back else: self.ticks = self._update_window(self.ticks, tick) # bars = tools.resample(self.ticks, self.resolution) bars = tools.resample(self.ticks, self.resolution, tz=self.timezone) if len(bars.index) > self.tick_bar_count > 0 or stale_tick: self.record_ts = tick.index[0] self._base_bar_handler(bars[bars['symbol'] == symbol][-1:]) window = int("".join([s for s in self.resolution if s.isdigit()])) if self.threads == 0: self.ticks = self._get_window_per_symbol(self.ticks, window) else: self_ticks = self._get_window_per_symbol(self_ticks, window) self.ticks = self._thread_safe_merge(symbol, self.ticks, self_ticks) # assign back self.tick_bar_count = len(bars.index) # record non time-based bars self.record(bars[-1:]) if not stale_tick: self.on_tick(self.get_instrument(tick))
def history(self, symbols, start, end=None, resolution="1T", tz="UTC"): # load runtime/default data if isinstance(symbols, str): symbols = symbols.split(',') # work with symbol groups symbols = list(map(_gen_symbol_group, symbols)) # convert datetime to string for MySQL try: start = start.strftime( ibDataTypes["DATE_TIME_FORMAT_LONG_MILLISECS"]) except: pass if end is not None: try: end = end.strftime( ibDataTypes["DATE_TIME_FORMAT_LONG_MILLISECS"]) except: pass # connect to mysql self.mysql_connect() # --- build query table = 'ticks' if ("K" in resolution) | ("V" in resolution) | ( "S" in resolution) else 'bars' query = """SELECT tbl.*, CONCAT(s.`symbol`, "_", s.`asset_class`) as symbol, s.symbol_group, s.asset_class, s.expiry FROM `{TABLE}` tbl LEFT JOIN `symbols` s ON tbl.symbol_id = s.id WHERE (`datetime` >= "{START}"{END_SQL}) """.replace( '{START}', start).replace('{TABLE}', table) if end is not None: query = query.replace('{END_SQL}', ' AND `datetime` <= "{END}"') query = query.replace('{END}', end) else: query = query.replace('{END_SQL}', '') if symbols[0].strip() != "*": query += """ AND ( s.`symbol_group` in ("{SYMBOLS}") OR s.`symbol` IN ("{SYMBOLS}") ) """ query = query.replace('{SYMBOLS}', '","'.join(symbols)) # --- end build query # get data using pandas data = pd.read_sql(query, self.dbconn).dropna() data.set_index('datetime', inplace=True) data.index = pd.to_datetime(data.index, utc=True) data['expiry'] = pd.to_datetime(data['expiry'], utc=True) if "K" not in resolution and "S" not in resolution: # construct continous contracts for futures all_dfs = [data[data['asset_class'] != 'FUT']] # generate dict of df per future futures_symbol_groups = list( data[data['asset_class'] == 'FUT']['symbol_group'].unique()) for key in futures_symbol_groups: future_group = data[data['symbol_group'] == key] continous = futures.create_continous_contract( future_group, resolution) all_dfs.append(continous) # make one df again data = pd.concat(all_dfs) data = tools.resample(data, resolution, tz) return data
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR", resample="1T"): """ Converts given DataFrame to a QTPyLib-compatible format and timezone :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) data : pd.DataFrame Pandas DataDrame with that instrument's market data output_path : str Path to where the resulting CSV should be saved (optional) index : pd.Series Pandas Series that will be used for df's index (optioanl) colsmap : dict Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's) kind : str Is this ``BAR`` or ``TICK`` data resample : str Pandas resolution (defaults to 1min/1T) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _TICKS_COLSMAP, _BARS_COLSMAP # work on copy df = data.copy() # ezibpy's csv? if set(df.columns) == set([ 'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']): df.rename(columns={ 'datetime': 'datetime', 'O': 'open', 'H': 'high', 'L': 'low', 'C': 'close', 'OI': 'volume', }, inplace=True) df.index = pd.to_datetime(df['datetime']) df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC") index = None # lower case columns df.columns = map(str.lower, df.columns) # set index if index is None: index = df.index # set defaults columns if not isinstance(colsmap, dict): colsmap = {} _colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP for el in _colsmap: if el not in colsmap: colsmap[el] = _colsmap[el] # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) # create contract string (no need for connection) ibConn = ezIBpy() contract_string = ibConn.contractString(instrument) asset_class = tools.gen_asset_class(contract_string) symbol_group = tools.gen_symbol_group(contract_string) # add symbol data df.loc[:, 'symbol'] = contract_string df.loc[:, 'symbol_group'] = symbol_group df.loc[:, 'asset_class'] = asset_class # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # rename columns to map df.rename(columns=colsmap, inplace=True) # force option columns on options if asset_class == "OPT": df = tools.force_options_columns(df) # remove all other columns known_cols = list(colsmap.values()) + \ ['symbol', 'symbol_group', 'asset_class', 'expiry'] for col in df.columns: if col not in known_cols: df.drop(col, axis=1, inplace=True) # set UTC index df.index = pd.to_datetime(index) df = tools.set_timezone(df, "UTC") df.index.rename("datetime", inplace=True) # resample if resample and kind == "BAR": df = tools.resample(df, resolution=resample, tz="UTC") # add expiry df.loc[:, 'expiry'] = np.nan if asset_class in ("FUT", "OPT", "FOP"): df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string) # save csv if output_path is not None: output_path = output_path[ :-1] if output_path.endswith('/') else output_path df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind)) # return df return df