def add_stale_tick(self):
if len(self.ticks) > 0:
tick = self.ticks[-1:].to_dict(orient='records')[0]
tick['timestamp'] = datetime.utcnow()
tick = pd.DataFrame(index=[0], data=tick)
tick.set_index('timestamp', inplace=True)
tick = tools.set_timezone(tick, tz=self.timezone)
self._tick_handler(tick, stale_tick=True)
def add_stale_tick(self):
ticks = self.ticks.copy()
if len(self.ticks.index) > 0:
last_tick_sec = float(tools.datetime64_to_datetime(ticks.index.values[-1]).strftime('%M.%S'))
for sym in list(self.ticks["symbol"].unique()):
tick = ticks[ticks['symbol']==sym][-5:].to_dict(orient='records')[-1]
tick['timestamp'] = datetime.utcnow()
if last_tick_sec != float(tick['timestamp'].strftime("%M.%S")):
tick = pd.DataFrame(index=[0], data=tick)
tick.set_index('timestamp', inplace=True)
tick = tools.set_timezone(tick, tz=self.timezone)
tick.loc[:, 'lastsize'] = 0 # no real size
self._tick_handler(tick, stale_tick=True)
def add_stale_tick(self):
ticks = self.ticks.copy()
if len(self.ticks.index) > 0:
last_tick_sec = float(tools.datetime64_to_datetime(
ticks.index.values[-1]).strftime('%M.%S'))
for sym in list(self.ticks["symbol"].unique()):
tick = ticks[ticks['symbol'] == sym][-5:].to_dict(orient='records')[-1]
tick['timestamp'] = datetime.utcnow()
if last_tick_sec != float(tick['timestamp'].strftime("%M.%S")):
tick = pd.DataFrame(index=[0], data=tick)
tick.set_index('timestamp', inplace=True)
tick = tools.set_timezone(tick, tz=self.timezone)
tick.loc[:, 'lastsize'] = 0 # no real size
self._tick_handler(tick, stale_tick=True)
def get_data_yahoo_intraday(symbol):
"""
Import intraday data (1M) from Yahoo finance (2 weeks max)
:Parameters:
symbol : str
symbol to downlaod intraday data for
:Returns:
data : pd.DataFrame
Pandas DataFrame with 1-minute bar data
"""
raw = requests.get("http://chartapi.finance.yahoo.com/instrument/1.0/" +
symbol + "/chartdata;type=quote;range=10d/csv")
cols = raw.text.split("values:")[1].split("\n")[0].lower()
data = raw.text.split("volume:")[1].split("\n")
data = "\n".join(data[1:])
if "timestamp" in cols:
df = pd.read_csv(StringIO(cols + "\n" + data),
index_col=["timestamp"],
parse_dates=["timestamp"])
df.index = pd.to_datetime(df.index, unit='s')
timezone = raw.text.split("timezone:")[1].split("\n")[0]
df = tools.set_timezone(df, timezone)
df = df.resample("T").last().dropna()
df['volume'] = df['volume'].astype(int)
return np.round(df, 2)
# parse csv
df = pd.read_csv(StringIO(cols + "\n" + data),
index_col=["date"],
parse_dates=["date"])
df.index = pd.to_datetime(df.index, utc=True)
# round
decimals = pd.Series([2, 2, 2, 2], index=['open', 'high', 'low', 'close'])
df = df.round(decimals)
df.loc[:, "symbol"] = symbol
return df[["symbol", "open", "high", "low", "close", "volume"]]
def get_data_yahoo_intraday(symbol, *args, **kwargs):
"""
Import intraday data (1M) from Yahoo finance (2 weeks max)
:Parameters:
symbol : str
symbol to downlaod intraday data for
:Returns:
data : pd.DataFrame
Pandas DataFrame with 1-minute bar data
"""
raw = requests.get("http://chartapi.finance.yahoo.com/instrument/1.0/" +
symbol + "/chartdata;type=quote;range=10d/csv")
cols = raw.text.split("values:")[1].split("\n")[0].lower()
data = raw.text.split("volume:")[1].split("\n")
data = "\n".join(data[1:])
if "timestamp" in cols:
df = pd.read_csv(StringIO(cols + "\n" + data),
index_col=["timestamp"], parse_dates=["timestamp"])
df.index = pd.to_datetime(df.index, unit='s')
timezone = raw.text.split("timezone:")[1].split("\n")[0]
df = tools.set_timezone(df, timezone)
df = df.resample("T").last().dropna()
df['volume'] = df['volume'].astype(int)
return np.round(df, 2)
# parse csv
df = pd.read_csv(StringIO(cols + "\n" + data),
index_col=["date"], parse_dates=["date"])
df.index = pd.to_datetime(df.index, utc=True)
# round
decimals = pd.Series([2, 2, 2, 2], index=['open', 'high', 'low', 'close'])
df = df.round(decimals)
df.loc[:, "symbol"] = symbol
return df[["symbol", "open", "high", "low", "close", "volume"]]
def prepare_data(instrument, data, output_path=None,
index=None, colsmap=None, kind="BAR", resample="1T"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to where the resulting CSV should be saved (optional)
index : pd.Series
Pandas Series that will be used for df's index (optioanl)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib
(default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
resample : str
Pandas resolution (defaults to 1min/1T)
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _TICKS_COLSMAP, _BARS_COLSMAP
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set([
'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# lower case columns
df.columns = map(str.lower, df.columns)
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + \
['symbol', 'symbol_group', 'asset_class', 'expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# resample
if resample and kind == "BAR":
df = tools.resample(df, resolution=resample, tz="UTC")
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR"):
"""
Converts given DataFrame to a QTPyLib-compatible format and timezone
:Parameters:
instrument : mixed
IB contract tuple / string (same as that given to strategy)
data : pd.DataFrame
Pandas DataDrame with that instrument's market data
output_path : str
Path to the location where the resulting CSV should be saved (default: ``None``)
index : pd.Series
Pandas Series that will be used for df's index (default is to use df.index)
colsmap : dict
Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's)
kind : str
Is this ``BAR`` or ``TICK`` data
:Returns:
data : pd.DataFrame
Pandas DataFrame in a QTPyLib-compatible format and timezone
"""
global _bars_colsmap, _ticks_colsmap
# work on copy
df = data.copy()
# ezibpy's csv?
if set(df.columns) == set(['datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']):
df.rename(columns={
'datetime': 'datetime',
'O': 'open',
'H': 'high',
'L': 'low',
'C': 'close',
'OI': 'volume',
}, inplace=True)
df.index = pd.to_datetime(df['datetime'])
df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC")
index = None
# set index
if index is None:
index = df.index
# set defaults columns
if not isinstance(colsmap, dict):
colsmap = {}
_colsmap = _ticks_colsmap if kind == "TICK" else _bars_colsmap
for el in _colsmap:
if el not in colsmap:
colsmap[el] = _colsmap[el]
# generate a valid ib tuple
instrument = tools.create_ib_tuple(instrument)
# create contract string (no need for connection)
ibConn = ezIBpy()
contract_string = ibConn.contractString(instrument)
asset_class = tools.gen_asset_class(contract_string)
symbol_group = tools.gen_symbol_group(contract_string)
# add symbol data
df.loc[:, 'symbol'] = contract_string
df.loc[:, 'symbol_group'] = symbol_group
df.loc[:, 'asset_class'] = asset_class
# validate columns
valid_cols = validate_columns(df, kind)
if not valid_cols:
raise ValueError('Invalid Column list')
# rename columns to map
df.rename(columns=colsmap, inplace=True)
# force option columns on options
if asset_class == "OPT":
df = tools.force_options_columns(df)
# remove all other columns
known_cols = list(colsmap.values()) + ['symbol','symbol_group','asset_class','expiry']
for col in df.columns:
if col not in known_cols:
df.drop(col, axis=1, inplace=True)
# set UTC index
df.index = pd.to_datetime(index)
df = tools.set_timezone(df, "UTC")
df.index.rename("datetime", inplace=True)
# add expiry
df.loc[:, 'expiry'] = np.nan
if asset_class in ("FUT", "OPT", "FOP"):
df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string)
# save csv
if output_path is not None:
output_path = output_path[
:-1] if output_path.endswith('/') else output_path
df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind))
# return df
return df
