def add_stale_tick(self): if len(self.ticks) > 0: tick = self.ticks[-1:].to_dict(orient='records')[0] tick['timestamp'] = datetime.utcnow() tick = pd.DataFrame(index=[0], data=tick) tick.set_index('timestamp', inplace=True) tick = tools.set_timezone(tick, tz=self.timezone) self._tick_handler(tick, stale_tick=True)
def add_stale_tick(self): ticks = self.ticks.copy() if len(self.ticks.index) > 0: last_tick_sec = float(tools.datetime64_to_datetime(ticks.index.values[-1]).strftime('%M.%S')) for sym in list(self.ticks["symbol"].unique()): tick = ticks[ticks['symbol']==sym][-5:].to_dict(orient='records')[-1] tick['timestamp'] = datetime.utcnow() if last_tick_sec != float(tick['timestamp'].strftime("%M.%S")): tick = pd.DataFrame(index=[0], data=tick) tick.set_index('timestamp', inplace=True) tick = tools.set_timezone(tick, tz=self.timezone) tick.loc[:, 'lastsize'] = 0 # no real size self._tick_handler(tick, stale_tick=True)
def add_stale_tick(self): ticks = self.ticks.copy() if len(self.ticks.index) > 0: last_tick_sec = float(tools.datetime64_to_datetime( ticks.index.values[-1]).strftime('%M.%S')) for sym in list(self.ticks["symbol"].unique()): tick = ticks[ticks['symbol'] == sym][-5:].to_dict(orient='records')[-1] tick['timestamp'] = datetime.utcnow() if last_tick_sec != float(tick['timestamp'].strftime("%M.%S")): tick = pd.DataFrame(index=[0], data=tick) tick.set_index('timestamp', inplace=True) tick = tools.set_timezone(tick, tz=self.timezone) tick.loc[:, 'lastsize'] = 0 # no real size self._tick_handler(tick, stale_tick=True)
def get_data_yahoo_intraday(symbol): """ Import intraday data (1M) from Yahoo finance (2 weeks max) :Parameters: symbol : str symbol to downlaod intraday data for :Returns: data : pd.DataFrame Pandas DataFrame with 1-minute bar data """ raw = requests.get("http://chartapi.finance.yahoo.com/instrument/1.0/" + symbol + "/chartdata;type=quote;range=10d/csv") cols = raw.text.split("values:")[1].split("\n")[0].lower() data = raw.text.split("volume:")[1].split("\n") data = "\n".join(data[1:]) if "timestamp" in cols: df = pd.read_csv(StringIO(cols + "\n" + data), index_col=["timestamp"], parse_dates=["timestamp"]) df.index = pd.to_datetime(df.index, unit='s') timezone = raw.text.split("timezone:")[1].split("\n")[0] df = tools.set_timezone(df, timezone) df = df.resample("T").last().dropna() df['volume'] = df['volume'].astype(int) return np.round(df, 2) # parse csv df = pd.read_csv(StringIO(cols + "\n" + data), index_col=["date"], parse_dates=["date"]) df.index = pd.to_datetime(df.index, utc=True) # round decimals = pd.Series([2, 2, 2, 2], index=['open', 'high', 'low', 'close']) df = df.round(decimals) df.loc[:, "symbol"] = symbol return df[["symbol", "open", "high", "low", "close", "volume"]]
def get_data_yahoo_intraday(symbol, *args, **kwargs): """ Import intraday data (1M) from Yahoo finance (2 weeks max) :Parameters: symbol : str symbol to downlaod intraday data for :Returns: data : pd.DataFrame Pandas DataFrame with 1-minute bar data """ raw = requests.get("http://chartapi.finance.yahoo.com/instrument/1.0/" + symbol + "/chartdata;type=quote;range=10d/csv") cols = raw.text.split("values:")[1].split("\n")[0].lower() data = raw.text.split("volume:")[1].split("\n") data = "\n".join(data[1:]) if "timestamp" in cols: df = pd.read_csv(StringIO(cols + "\n" + data), index_col=["timestamp"], parse_dates=["timestamp"]) df.index = pd.to_datetime(df.index, unit='s') timezone = raw.text.split("timezone:")[1].split("\n")[0] df = tools.set_timezone(df, timezone) df = df.resample("T").last().dropna() df['volume'] = df['volume'].astype(int) return np.round(df, 2) # parse csv df = pd.read_csv(StringIO(cols + "\n" + data), index_col=["date"], parse_dates=["date"]) df.index = pd.to_datetime(df.index, utc=True) # round decimals = pd.Series([2, 2, 2, 2], index=['open', 'high', 'low', 'close']) df = df.round(decimals) df.loc[:, "symbol"] = symbol return df[["symbol", "open", "high", "low", "close", "volume"]]
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR", resample="1T"): """ Converts given DataFrame to a QTPyLib-compatible format and timezone :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) data : pd.DataFrame Pandas DataDrame with that instrument's market data output_path : str Path to where the resulting CSV should be saved (optional) index : pd.Series Pandas Series that will be used for df's index (optioanl) colsmap : dict Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's) kind : str Is this ``BAR`` or ``TICK`` data resample : str Pandas resolution (defaults to 1min/1T) :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _TICKS_COLSMAP, _BARS_COLSMAP # work on copy df = data.copy() # ezibpy's csv? if set(df.columns) == set([ 'datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']): df.rename(columns={ 'datetime': 'datetime', 'O': 'open', 'H': 'high', 'L': 'low', 'C': 'close', 'OI': 'volume', }, inplace=True) df.index = pd.to_datetime(df['datetime']) df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC") index = None # lower case columns df.columns = map(str.lower, df.columns) # set index if index is None: index = df.index # set defaults columns if not isinstance(colsmap, dict): colsmap = {} _colsmap = _TICKS_COLSMAP if kind == "TICK" else _BARS_COLSMAP for el in _colsmap: if el not in colsmap: colsmap[el] = _colsmap[el] # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) # create contract string (no need for connection) ibConn = ezIBpy() contract_string = ibConn.contractString(instrument) asset_class = tools.gen_asset_class(contract_string) symbol_group = tools.gen_symbol_group(contract_string) # add symbol data df.loc[:, 'symbol'] = contract_string df.loc[:, 'symbol_group'] = symbol_group df.loc[:, 'asset_class'] = asset_class # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # rename columns to map df.rename(columns=colsmap, inplace=True) # force option columns on options if asset_class == "OPT": df = tools.force_options_columns(df) # remove all other columns known_cols = list(colsmap.values()) + \ ['symbol', 'symbol_group', 'asset_class', 'expiry'] for col in df.columns: if col not in known_cols: df.drop(col, axis=1, inplace=True) # set UTC index df.index = pd.to_datetime(index) df = tools.set_timezone(df, "UTC") df.index.rename("datetime", inplace=True) # resample if resample and kind == "BAR": df = tools.resample(df, resolution=resample, tz="UTC") # add expiry df.loc[:, 'expiry'] = np.nan if asset_class in ("FUT", "OPT", "FOP"): df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string) # save csv if output_path is not None: output_path = output_path[ :-1] if output_path.endswith('/') else output_path df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind)) # return df return df
def prepare_data(instrument, data, output_path=None, index=None, colsmap=None, kind="BAR"): """ Converts given DataFrame to a QTPyLib-compatible format and timezone :Parameters: instrument : mixed IB contract tuple / string (same as that given to strategy) data : pd.DataFrame Pandas DataDrame with that instrument's market data output_path : str Path to the location where the resulting CSV should be saved (default: ``None``) index : pd.Series Pandas Series that will be used for df's index (default is to use df.index) colsmap : dict Dict for mapping df's columns to those used by QTPyLib (default assumes same naming convention as QTPyLib's) kind : str Is this ``BAR`` or ``TICK`` data :Returns: data : pd.DataFrame Pandas DataFrame in a QTPyLib-compatible format and timezone """ global _bars_colsmap, _ticks_colsmap # work on copy df = data.copy() # ezibpy's csv? if set(df.columns) == set(['datetime', 'C', 'H', 'L', 'O', 'OI', 'V', 'WAP']): df.rename(columns={ 'datetime': 'datetime', 'O': 'open', 'H': 'high', 'L': 'low', 'C': 'close', 'OI': 'volume', }, inplace=True) df.index = pd.to_datetime(df['datetime']) df.index = df.index.tz_localize(tools.get_timezone()).tz_convert("UTC") index = None # set index if index is None: index = df.index # set defaults columns if not isinstance(colsmap, dict): colsmap = {} _colsmap = _ticks_colsmap if kind == "TICK" else _bars_colsmap for el in _colsmap: if el not in colsmap: colsmap[el] = _colsmap[el] # generate a valid ib tuple instrument = tools.create_ib_tuple(instrument) # create contract string (no need for connection) ibConn = ezIBpy() contract_string = ibConn.contractString(instrument) asset_class = tools.gen_asset_class(contract_string) symbol_group = tools.gen_symbol_group(contract_string) # add symbol data df.loc[:, 'symbol'] = contract_string df.loc[:, 'symbol_group'] = symbol_group df.loc[:, 'asset_class'] = asset_class # validate columns valid_cols = validate_columns(df, kind) if not valid_cols: raise ValueError('Invalid Column list') # rename columns to map df.rename(columns=colsmap, inplace=True) # force option columns on options if asset_class == "OPT": df = tools.force_options_columns(df) # remove all other columns known_cols = list(colsmap.values()) + ['symbol','symbol_group','asset_class','expiry'] for col in df.columns: if col not in known_cols: df.drop(col, axis=1, inplace=True) # set UTC index df.index = pd.to_datetime(index) df = tools.set_timezone(df, "UTC") df.index.rename("datetime", inplace=True) # add expiry df.loc[:, 'expiry'] = np.nan if asset_class in ("FUT", "OPT", "FOP"): df.loc[:, 'expiry'] = contract_expiry_from_symbol(contract_string) # save csv if output_path is not None: output_path = output_path[ :-1] if output_path.endswith('/') else output_path df.to_csv("%s/%s.%s.csv" % (output_path, contract_string, kind)) # return df return df