def _update_window(self, df, data, window=None, resolution=None):
if df is None:
df = data
else:
df = df.append(data)
# resample
if resolution is not None:
try:
tz = str(df.index.tz)
except:
tz = None
df = tools.resample(df, resolution=resolution, tz=tz)
# remove duplicates rows
df.loc[:, '_idx_'] = df.index
df.drop_duplicates(
subset=['_idx_', 'symbol', 'symbol_group', 'asset_class'],
keep='last', inplace=True)
df.drop('_idx_', axis=1, inplace=True)
# return
if window is None:
return df
# return df[-window:]
return self._get_window_per_symbol(df, window)
def _tick_handler(self, tick, stale_tick=False):
self._cancel_expired_pending_orders()
# initial value
if self.record_ts is None:
self.record_ts = tick.index[0]
if self.resolution[-1] not in ("S", "K", "V"):
self.ticks = self._update_window(self.ticks, tick, window=self.tick_window)
else:
self.ticks = self._update_window(self.ticks, tick)
bars = tools.resample(self.ticks, self.resolution)
if len(bars.index) > self.tick_bar_count > 0 or stale_tick:
self.record_ts = tick.index[0]
self._bar_handler(bars)
periods = int("".join([s for s in self.resolution if s.isdigit()]))
self.ticks = self.ticks[-periods:]
self.tick_bar_count = len(bars)
# record tick bars
self.record(bars[-1:])
if not stale_tick:
self.on_tick(self.get_instrument(tick))
def _tick_handler(self, tick, stale_tick=False):
self._cancel_expired_pending_orders()
# tick symbol
symbol = tick['symbol'].values
if len(symbol) == 0:
return
symbol = symbol[0]
self.last_price[symbol] = float(tick['last'].values[0])
# work on copy
self_ticks = self.ticks.copy()
# initial value
if self.record_ts is None:
self.record_ts = tick.index[0]
if self.resolution[-1] not in ("S", "K", "V"):
if self.threads == 0:
self.ticks = self._update_window(
self.ticks, tick, window=self.tick_window)
else:
self_ticks = self._update_window(
self_ticks, tick, window=self.tick_window)
self.ticks = self._thread_safe_merge(
symbol, self.ticks, self_ticks) # assign back
else:
self.ticks = self._update_window(self.ticks, tick)
# bars = tools.resample(self.ticks, self.resolution)
bars = tools.resample(
self.ticks, self.resolution, tz=self.timezone)
if len(bars.index) > self.tick_bar_count > 0 or stale_tick:
self.record_ts = tick.index[0]
self._base_bar_handler(bars[bars['symbol'] == symbol][-1:])
window = int(
"".join([s for s in self.resolution if s.isdigit()]))
if self.threads == 0:
self.ticks = self._get_window_per_symbol(
self.ticks, window)
else:
self_ticks = self._get_window_per_symbol(
self_ticks, window)
self.ticks = self._thread_safe_merge(
symbol, self.ticks, self_ticks) # assign back
self.tick_bar_count = len(bars.index)
# record non time-based bars
self.record(bars[-1:])
if not stale_tick:
if self.ticks[(self.ticks['symbol'] == symbol) | (
self.ticks['symbol_group'] == symbol)].empty:
return
tick_instrument = self.get_instrument(tick)
if tick_instrument:
self.on_tick(tick_instrument)
def _update_window(self, df, data, window=None, resolution=None):
if df is None:
df = data
else:
df = df.append(data)
if resolution is not None:
try:
tz = str(df.index.tz)
except:
tz = None
df = tools.resample(df, resolution=resolution, tz=tz)
if window is None:
return df
return df[-window:]
def _tick_handler(self, tick):
self._cancel_expired_pending_orders()
if "K" not in self.resolution and "V" not in self.resolution:
self.ticks = self._update_window(self.ticks, tick, window=self.tick_window)
else:
self.ticks = self._update_window(self.ticks, tick)
bar = tools.resample(self.ticks, self.resolution)
if len(bar.index) > self.tick_bar_count > 0:
self.record_ts = tick.index[0]
self._bar_handler(bar)
periods = int("".join([s for s in self.resolution if s.isdigit()]))
self.ticks = self.ticks[-periods:]
self.tick_bar_count = len(bar)
# record tick bar
self.record(bar)
self.on_tick(self.get_instrument(tick))
def _tick_handler(self, tick, stale_tick=False):
self._cancel_expired_pending_orders()
# tick symbol
symbol = tick['symbol'].values[0]
self_ticks = self.ticks.copy() # work on copy
# initial value
if self.record_ts is None:
self.record_ts = tick.index[0]
if self.resolution[-1] not in ("S", "K", "V"):
if self.threads == 0:
self.ticks = self._update_window(self.ticks, tick, window=self.tick_window)
else:
self_ticks = self._update_window(self_ticks, tick, window=self.tick_window)
self.ticks = self._thread_safe_merge(symbol, self.ticks, self_ticks) # assign back
else:
self.ticks = self._update_window(self.ticks, tick)
# bars = tools.resample(self.ticks, self.resolution)
bars = tools.resample(self.ticks, self.resolution, tz=self.timezone)
if len(bars.index) > self.tick_bar_count > 0 or stale_tick:
self.record_ts = tick.index[0]
self._base_bar_handler(bars[bars['symbol'] == symbol][-1:])
window = int("".join([s for s in self.resolution if s.isdigit()]))
if self.threads == 0:
self.ticks = self._get_window_per_symbol(self.ticks, window)
else:
self_ticks = self._get_window_per_symbol(self_ticks, window)
self.ticks = self._thread_safe_merge(symbol, self.ticks, self_ticks) # assign back
self.tick_bar_count = len(bars.index)
# record non time-based bars
self.record(bars[-1:])
if not stale_tick:
self.on_tick(self.get_instrument(tick))
def history(self, symbols, start, end=None, resolution="1T", tz="UTC"):
# load runtime/default data
if isinstance(symbols, str):
symbols = symbols.split(',')
# work with symbol groups
symbols = list(map(_gen_symbol_group, symbols))
# convert datetime to string for MySQL
try:
start = start.strftime(
ibDataTypes["DATE_TIME_FORMAT_LONG_MILLISECS"])
except:
pass
if end is not None:
try:
end = end.strftime(
ibDataTypes["DATE_TIME_FORMAT_LONG_MILLISECS"])
except:
pass
# connect to mysql
self.mysql_connect()
# --- build query
table = 'ticks' if ("K" in resolution) | ("V" in resolution) | (
"S" in resolution) else 'bars'
query = """SELECT tbl.*,
CONCAT(s.`symbol`, "_", s.`asset_class`) as symbol, s.symbol_group, s.asset_class, s.expiry
FROM `{TABLE}` tbl LEFT JOIN `symbols` s ON tbl.symbol_id = s.id
WHERE (`datetime` >= "{START}"{END_SQL}) """.replace(
'{START}', start).replace('{TABLE}', table)
if end is not None:
query = query.replace('{END_SQL}', ' AND `datetime` <= "{END}"')
query = query.replace('{END}', end)
else:
query = query.replace('{END_SQL}', '')
if symbols[0].strip() != "*":
query += """ AND ( s.`symbol_group` in ("{SYMBOLS}") OR s.`symbol` IN ("{SYMBOLS}") ) """
query = query.replace('{SYMBOLS}', '","'.join(symbols))
# --- end build query
# get data using pandas
data = pd.read_sql(query, self.dbconn).dropna()
data.set_index('datetime', inplace=True)
data.index = pd.to_datetime(data.index, utc=True)
data['expiry'] = pd.to_datetime(data['expiry'], utc=True)
if "K" not in resolution and "S" not in resolution:
# construct continous contracts for futures
all_dfs = [data[data['asset_class'] != 'FUT']]
# generate dict of df per future
futures_symbol_groups = list(
data[data['asset_class'] == 'FUT']['symbol_group'].unique())
for key in futures_symbol_groups:
future_group = data[data['symbol_group'] == key]
continous = futures.create_continous_contract(
future_group, resolution)
all_dfs.append(continous)
# make one df again
data = pd.concat(all_dfs)
data = tools.resample(data, resolution, tz)
return data
def prepare_data(instrument, data, output_path=None,
index=None, colsmap=None, kind="BAR", resample="1T"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to where the resulting CSV should be saved (optional)
index : pd.Series
Pandas Series that will be used for df's index (optioanl)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib
(default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
resample : str
Pandas resolution (defaults to 1min/1T)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _TICKS_COLSMAP, _BARS_COLSMAP
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set([
'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# lower case columns
df.columns = map(str.lower, df.columns)
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + \
['symbol', 'symbol_group', 'asset_class', 'expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# resample
if resample and kind == "BAR":
df = tools.resample(df, resolution=resample, tz="UTC")
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
