def map_frequency(value, source='ccxt', raise_error=True): """ Map a frequency value between CCXT and Catalyst Parameters ---------- value: str source: str raise_error: bool Returns ------- Notes ----- The Pandas offset aliases supported by Catalyst: Alias Description W weekly frequency M month end frequency D calendar day frequency H hourly frequency T, min minutely frequency The CCXT timeframes: '1m': '1minute', '1h': '1hour', '1d': '1day', '1w': '1week', '1M': '1month', '1y': '1year', """ match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value, re.M | re.I) if match: candle_size = int(match.group(1)) \ if match.group(1) else 1 unit = match.group(2) else: raise ValueError('Unable to parse frequency or timeframe') if source == 'ccxt': if unit == 'd': result = '{}D'.format(candle_size) elif unit == 'm': result = '{}T'.format(candle_size) elif unit == 'h': result = '{}H'.format(candle_size) elif unit == 'w': result = '{}W'.format(candle_size) elif unit == 'M': result = '{}M'.format(candle_size) elif raise_error: raise InvalidHistoryTimeframeError(timeframe=value) else: if unit == 'D': result = '{}d'.format(candle_size) elif unit == 'min' or unit == 'T': result = '{}m'.format(candle_size) elif unit == 'H': result = '{}h'.format(candle_size) elif unit == 'W': result = '{}w'.format(candle_size) elif unit == 'M': result = '{}M'.format(candle_size) elif raise_error: raise InvalidHistoryFrequencyError(frequency=value) return result
def get_candles(self, freq, assets, bar_count=None, start_dt=None, end_dt=None): """ Retrieve OHLVC candles from Poloniex :param freq: :param assets: :param bar_count: :return: Available Frequencies --------------------- '5m', '15m', '30m', '2h', '4h', '1D' """ if end_dt is None: end_dt = pd.Timestamp.utcnow() log.debug('retrieving {bars} {freq} candles on {exchange} from ' '{end_dt} for markets {symbols}, '.format( bars=bar_count, freq=freq, exchange=self.name, end_dt=end_dt, symbols=get_symbols_string(assets))) if freq == '1T' and (bar_count == 1 or bar_count is None): # TODO: use the order book instead # We use the 5m to fetch the last bar frequency = 300 elif freq == '5T': frequency = 300 elif freq == '15T': frequency = 900 elif freq == '30T': frequency = 1800 elif freq == '120T': frequency = 7200 elif freq == '240T': frequency = 14400 elif freq == '1D': frequency = 86400 else: # Poloniex does not offer 1m data candles # It is likely to error out there frequently raise InvalidHistoryFrequencyError(frequency=freq) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets ohlc_map = dict() for asset in asset_list: delta = end_dt - pd.to_datetime('1970-1-1', utc=True) end = int(delta.total_seconds()) if bar_count is None: start = end - 2 * frequency else: start = end - bar_count * frequency try: response = self.api.returnchartdata(self.get_symbol(asset), frequency, start, end) except Exception as e: raise ExchangeRequestError(error=e) if 'error' in response: raise ExchangeRequestError( error='Unable to retrieve candles: {}'.format( response.content)) def ohlc_from_candle(candle): last_traded = pd.Timestamp.utcfromtimestamp(candle['date']) last_traded = last_traded.replace(tzinfo=pytz.UTC) ohlc = dict(open=np.float64(candle['open']), high=np.float64(candle['high']), low=np.float64(candle['low']), close=np.float64(candle['close']), volume=np.float64(candle['volume']), price=np.float64(candle['close']), last_traded=last_traded) return ohlc if bar_count is None: ohlc_map[asset] = ohlc_from_candle(response[0]) else: ohlc_bars = [] for candle in response: ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']): """ Takes an arbitrary candle size (e.g. 15T) and converts to the lowest common denominator supported by the data bundles (e.g. 1T). The data bundles only support 1T and 1D frequencies. If another frequency is requested, Catalyst must request the underlying data and resample. Notes ----- We're trying to use Pandas convention for frequency aliases. Parameters ---------- freq: str data_frequency: str Returns ------- str, int, str, str """ if data_frequency is None: data_frequency = 'daily' if freq.upper().endswith('D') else 'minute' if freq == 'minute': unit = 'T' candle_size = 1 elif freq == 'daily': unit = 'D' candle_size = 1 else: freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I) if freq_match: candle_size = int(freq_match.group(1)) if freq_match.group(1) \ else 1 unit = freq_match.group(2) else: raise InvalidHistoryFrequencyError(frequency=freq) # TODO: some exchanges support H and W frequencies but not bundles # Find a way to pass-through these parameters to exchanges # but resample from minute or daily in backtest mode # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between # Pandas offet aliases (used by Catalyst) and the CCXT timeframes if unit.lower() == 'd': unit = 'D' alias = '{}D'.format(candle_size) if data_frequency == 'minute': data_frequency = 'daily' elif unit.lower() == 'm' or unit == 'T': unit = 'T' alias = '{}T'.format(candle_size) data_frequency = 'minute' elif unit.lower() == 'h': data_frequency = 'minute' if 'H' in supported_freqs: unit = 'H' alias = '{}H'.format(candle_size) else: candle_size = candle_size * 60 alias = '{}T'.format(candle_size) else: raise InvalidHistoryFrequencyAlias(freq=freq) return alias, candle_size, unit, data_frequency
def get_candles(self, data_frequency, assets, bar_count=None, start_dt=None, end_dt=None): """ Retrieve OHLVC candles from Poloniex :param data_frequency: :param assets: :param bar_count: :return: Available Frequencies --------------------- '5m', '15m', '30m', '2h', '4h', '1D' """ # TODO: implement end_dt and start_dt filters if (data_frequency == '5m' or data_frequency == 'minute'): # TODO: Polo does not have '1m' frequency = 300 elif (data_frequency == '15m'): frequency = 900 elif (data_frequency == '30m'): frequency = 1800 elif (data_frequency == '2h'): frequency = 7200 elif (data_frequency == '4h'): frequency = 14400 elif (data_frequency == '1D' or data_frequency == 'daily'): frequency = 86400 else: raise InvalidHistoryFrequencyError(frequency=data_frequency) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets ohlc_map = dict() for asset in asset_list: end = int(time.time()) if (bar_count is None): start = end - 2 * frequency else: start = end - bar_count * frequency try: response = self.api.returnchartdata(self.get_symbol(asset), frequency, start, end) except Exception as e: raise ExchangeRequestError(error=e) if 'error' in response: raise ExchangeRequestError( error='Unable to retrieve candles: {}'.format( response.content)) def ohlc_from_candle(candle): last_traded = pd.Timestamp.utcfromtimestamp(candle['date']) last_traded = last_traded.replace(tzinfo=pytz.UTC) ohlc = dict(open=np.float64(candle['open']), high=np.float64(candle['high']), low=np.float64(candle['low']), close=np.float64(candle['close']), volume=np.float64(candle['volume']), price=np.float64(candle['close']), last_traded=last_traded) return ohlc if bar_count is None: ohlc_map[asset] = ohlc_from_candle(response[0]) else: ohlc_bars = [] for candle in response: ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self, data_frequency, assets, bar_count=None): """ Retrieve OHLVC candles from Bitfinex :param data_frequency: :param assets: :param bar_count: :return: Available Frequencies --------------------- '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' """ # TODO: use BcolzMinuteBarReader to read from cache freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I) if freq_match: number = int(freq_match.group(1)) unit = freq_match.group(2) if unit == 'd': converted_unit = 'D' else: converted_unit = unit frequency = '{}{}'.format(number, converted_unit) allowed_frequencies = [ '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' ] if frequency not in allowed_frequencies: raise InvalidHistoryFrequencyError(frequency=data_frequency) elif data_frequency == 'minute': frequency = '1m' elif data_frequency == 'daily': frequency = '1D' else: raise InvalidHistoryFrequencyError(frequency=data_frequency) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets ohlc_map = dict() for asset in asset_list: symbol = self._get_v2_symbol(asset) url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format( url=self.url, frequency=frequency, symbol=symbol) if bar_count: is_list = True url += '/hist?limit={}'.format(int(bar_count)) else: is_list = False url += '/last' try: response = requests.get(url) except Exception as e: raise ExchangeRequestError(error=e) if 'error' in response.content: raise ExchangeRequestError( error='Unable to retrieve candles: {}'.format( response.content)) candles = response.json() def ohlc_from_candle(candle): ohlc = dict(open=np.float64(candle[1]), high=np.float64(candle[3]), low=np.float64(candle[4]), close=np.float64(candle[2]), volume=np.float64(candle[5]), price=np.float64(candle[2]), last_traded=pd.Timestamp.utcfromtimestamp( candle[0] / 1000.0)) return ohlc if is_list: ohlc_bars = [] # We can to list candles from old to new for candle in reversed(candles): ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars else: ohlc = ohlc_from_candle(candles) ohlc_map[asset] = ohlc return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_frequency(freq, data_frequency): """ Get the frequency parameters. Notes ----- We're trying to use Pandas convention for frequency aliases. Parameters ---------- freq: str data_frequency: str Returns ------- str, int, str, str """ if freq == 'minute': unit = 'T' candle_size = 1 elif freq == 'daily': unit = 'D' candle_size = 1 else: freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I) if freq_match: candle_size = int(freq_match.group(1)) if freq_match.group(1) \ else 1 unit = freq_match.group(2) else: raise InvalidHistoryFrequencyError(frequency=freq) # TODO: some exchanges support H and W frequencies but not bundles # Find a way to pass-through these parameters to exchanges # but resample from minute or daily in backtest mode # see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between # Pandas offet aliases (used by Catalyst) and the CCXT timeframes if unit.lower() == 'd': alias = '{}D'.format(candle_size) if data_frequency == 'minute': data_frequency = 'daily' elif unit.lower() == 'm' or unit == 'T': alias = '{}T'.format(candle_size) if data_frequency == 'daily': data_frequency = 'minute' # elif unit.lower() == 'h': # candle_size = candle_size * 60 # # alias = '{}T'.format(candle_size) # if data_frequency == 'daily': # data_frequency = 'minute' else: raise InvalidHistoryFrequencyAlias(freq=freq) return alias, candle_size, unit, data_frequency
def get_candles(self, freq, assets, bar_count=None, start_dt=None, end_dt=None): """ Supported Intervals ------------------- day, oneMin, fiveMin, thirtyMin, hour :param freq: :param assets: :param bar_count: :param start_dt :param end_dt :return: """ # TODO: this has no effect at the moment if end_dt is None: end_dt = pd.Timestamp.utcnow() log.debug('retrieving {bars} {freq} candles on {exchange} from ' '{end_dt} for markets {symbols}, '.format( bars=bar_count, freq=freq, exchange=self.name, end_dt=end_dt, symbols=get_symbols_string(assets))) if freq == '1T': frequency = 'oneMin' elif freq == '5T': frequency = 'fiveMin' elif freq == '30T': frequency = 'thirtyMin' elif freq == '60T': frequency = 'hour' elif freq == '1D': frequency = 'day' else: raise InvalidHistoryFrequencyError(frequency=freq) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets for asset in asset_list: end = int(time.mktime(end_dt.timetuple())) url = '{url}/pub/market/GetTicks?marketName={symbol}' \ '&tickInterval={frequency}&_={end}'.format( url=URL2, symbol=self.get_symbol(asset), frequency=frequency, end=end, ) try: data = json.loads(urllib.request.urlopen(url).read().decode()) except Exception as e: raise ExchangeRequestError(error=e) if data['message']: raise ExchangeRequestError( error='Unable to fetch candles {}'.format(data['message'])) candles = data['result'] def ohlc_from_candle(candle): ohlc = dict(open=candle['O'], high=candle['H'], low=candle['L'], close=candle['C'], volume=candle['V'], price=candle['C'], last_traded=pd.to_datetime(candle['T'], utc=True)) return ohlc ordered_candles = list(reversed(candles)) ohlc_map = dict() if bar_count is None: ohlc_map[asset] = ohlc_from_candle(ordered_candles[0]) else: # TODO: optimize ohlc_bars = [] for candle in ordered_candles[:bar_count]: ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self, freq, assets, bar_count=None, start_dt=None, end_dt=None): """ Retrieve OHLVC candles from Bitfinex :param data_frequency: :param assets: :param bar_count: :return: Available Frequencies --------------------- '1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D', '1M' """ log.debug('retrieving {bars} {freq} candles on {exchange} from ' '{end_dt} for markets {symbols}, '.format( bars=bar_count, freq=freq, exchange=self.name, end_dt=end_dt, symbols=get_symbols_string(assets))) allowed_frequencies = [ '1T', '5T', '15T', '30T', '60T', '180T', '360T', '720T', '1D', '7D', '14D', '30D' ] if freq not in allowed_frequencies: raise InvalidHistoryFrequencyError(frequency=freq) freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I) if freq_match: number = int(freq_match.group(1)) unit = freq_match.group(2) if unit == 'T': if number in [60, 180, 360, 720]: number = number / 60 converted_unit = 'h' else: converted_unit = 'm' else: converted_unit = unit frequency = '{}{}'.format(number, converted_unit) else: raise InvalidHistoryFrequencyError(frequency=freq) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets ohlc_map = dict() for asset in asset_list: symbol = self._get_v2_symbol(asset) url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format( url=self.url, frequency=frequency, symbol=symbol) if bar_count: is_list = True url += '/hist?limit={}'.format(int(bar_count)) def get_ms(date): epoch = datetime.datetime.utcfromtimestamp(0) epoch = epoch.replace(tzinfo=pytz.UTC) return (date - epoch).total_seconds() * 1000.0 if start_dt is not None: start_ms = get_ms(start_dt) url += '&start={0:f}'.format(start_ms) if end_dt is not None: end_ms = get_ms(end_dt) url += '&end={0:f}'.format(end_ms) else: is_list = False url += '/last' try: self.ask_request() response = requests.get(url) except Exception as e: raise ExchangeRequestError(error=e) if 'error' in response.content: raise ExchangeRequestError( error='Unable to retrieve candles: {}'.format( response.content)) candles = response.json() def ohlc_from_candle(candle): last_traded = pd.Timestamp.utcfromtimestamp(candle[0] / 1000.0) last_traded = last_traded.replace(tzinfo=pytz.UTC) ohlc = dict(open=np.float64(candle[1]), high=np.float64(candle[3]), low=np.float64(candle[4]), close=np.float64(candle[2]), volume=np.float64(candle[5]), price=np.float64(candle[2]), last_traded=last_traded) return ohlc if is_list: ohlc_bars = [] # We can to list candles from old to new for candle in reversed(candles): ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars else: ohlc = ohlc_from_candle(candles) ohlc_map[asset] = ohlc return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self, data_frequency, assets, bar_count=None, start_date=None): """ Supported Intervals ------------------- day, oneMin, fiveMin, thirtyMin, hour :param data_frequency: :param assets: :param bar_count: :return: """ log.info('retrieving candles') if data_frequency == 'minute' or data_frequency == '1m': frequency = 'oneMin' elif data_frequency == '5m': frequency = 'fiveMin' elif data_frequency == '30m': frequency = 'thirtyMin' elif data_frequency == '1h': frequency = 'hour' elif data_frequency == 'daily' or data_frequency == '1D': frequency = 'day' else: raise InvalidHistoryFrequencyError(frequency=data_frequency) # Making sure that assets are iterable asset_list = [assets] if isinstance(assets, TradingPair) else assets ohlc_map = dict() for asset in asset_list: url = '{url}/pub/market/GetTicks?marketName={symbol}' \ '&tickInterval={frequency}&_=1499127220008'.format( url=URL2, symbol=self.get_symbol(asset), frequency=frequency ) try: data = json.loads(urllib.request.urlopen(url).read().decode()) except Exception as e: raise ExchangeRequestError(error=e) if data['message']: raise ExchangeRequestError( error='Unable to fetch candles {}'.format(data['message'])) candles = data['result'] def ohlc_from_candle(candle): ohlc = dict(open=candle['O'], high=candle['H'], low=candle['L'], close=candle['C'], volume=candle['V'], price=candle['C'], last_traded=pd.to_datetime(candle['T'], utc=True)) return ohlc ordered_candles = list(reversed(candles)) if bar_count is None: ohlc_map[asset] = ohlc_from_candle(ordered_candles[0]) else: ohlc_bars = [] for candle in ordered_candles[:bar_count]: ohlc = ohlc_from_candle(candle) ohlc_bars.append(ohlc) ohlc_map[asset] = ohlc_bars return ohlc_map[assets] \ if isinstance(assets, TradingPair) else ohlc_map
def get_history_window(self, assets, end_dt, bar_count, frequency, field, data_frequency=None, ffill=True): """ Public API method that returns a dataframe containing the requested history window. Data is fully adjusted. Parameters ---------- assets : list of catalyst.data.Asset objects The assets whose data is desired. end_dt: not applicable to cryptocurrencies bar_count: int The number of bars desired. frequency: string "1d" or "1m" field: string The desired field of the asset. data_frequency: string The frequency of the data to query; i.e. whether the data is 'daily' or 'minute' bars. # TODO: fill how? ffill: boolean Forward-fill missing values. Only has effect if field is 'price'. Returns ------- A dataframe containing the requested data. """ freq_match = re.match(r'([0-9].*)(m|M|d|D)', frequency, re.M | re.I) if freq_match: candle_size = int(freq_match.group(1)) unit = freq_match.group(2) else: raise InvalidHistoryFrequencyError(frequency) if unit.lower() == 'd': if data_frequency == 'minute': data_frequency = 'daily' elif unit.lower() == 'm': if data_frequency == 'daily': data_frequency = 'minute' else: raise InvalidHistoryFrequencyError(frequency) adj_bar_count = candle_size * bar_count try: series = self.bundle.get_history_window_series_and_load( assets=assets, end_dt=end_dt, bar_count=adj_bar_count, field=field, data_frequency=data_frequency) except PricingDataNotLoadedError: series = dict() for asset in assets: if asset not in series or series[asset].index[-1] < end_dt: # Adding bars too recent to be contained in the consolidated # exchanges bundles. We go directly against the exchange # to retrieve the candles. start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency) trailing_dt = \ series[asset].index[-1] + get_delta(1, data_frequency) \ if asset in series else start_dt trailing_bar_count = \ get_periods(trailing_dt, end_dt, data_frequency) # The get_history method supports multiple asset candles = self.get_candles(data_frequency=data_frequency, assets=asset, bar_count=trailing_bar_count, end_dt=end_dt) last_value = series[asset].iloc(0) if asset in series \ else np.nan candle_series = self.get_series_from_candles( candles=candles, start_dt=trailing_dt, end_dt=end_dt, field=field, previous_value=last_value) if asset in series: series[asset].append(candle_series) else: series[asset] = candle_series df = pd.DataFrame(series) if candle_size > 1: if field == 'open': agg = 'first' elif field == 'high': agg = 'max' elif field == 'low': agg = 'min' elif field == 'close': agg = 'last' elif field == 'volume': agg = 'sum' else: raise ValueError('Invalid field.') df = df.resample('{}T'.format(candle_size)).agg(agg) return df
def prepare_chunks(self, assets, data_frequency, start_dt, end_dt): """ Split a price data request into chunks corresponding to individual bundles. :param assets: :param data_frequency: :param start_dt: :param end_dt: :return: """ reader = self.get_reader(data_frequency) chunks = [] for asset in assets: try: asset_start, asset_end = \ get_adj_dates(start_dt, end_dt, [asset], data_frequency) except NoDataAvailableOnExchange: continue start_dt = max(start_dt, self.calendar.first_trading_session) start_dt = max(start_dt, asset_start) # Aligning start / end dates with the daily calendar sessions = get_periods_range(start_dt, end_dt, data_frequency) \ if data_frequency == 'minute' \ else self.calendar.sessions_in_range(start_dt, end_dt) if asset_start < sessions[0]: asset_start = sessions[0] if asset_end > sessions[-1]: asset_end = sessions[-1] chunk_labels = [] dt = sessions[0] while dt <= sessions[-1]: label = '{}-{:02d}'.format(dt.year, dt.month) \ if data_frequency == 'minute' else '{}'.format(dt.year) if label not in chunk_labels: chunk_labels.append(label) # Adjusting the period dates to match the availability # of the trading pair if data_frequency == 'minute': period_start, period_end = get_month_start_end(dt) asset_start_month, _ = get_month_start_end(asset_start) if asset_start_month == period_start \ and period_start < asset_start: period_start = asset_start _, asset_end_month = get_month_start_end(asset_end) if asset_end_month == period_end \ and period_end > asset_end: period_end = asset_end elif data_frequency == 'daily': period_start, period_end = get_year_start_end(dt) asset_start_year, _ = get_year_start_end(asset_start) if asset_start_year == period_start \ and period_start < asset_start: period_start = asset_start _, asset_end_year = get_year_start_end(asset_end) if asset_end_year == period_end \ and period_end > asset_end: period_end = asset_end else: raise InvalidHistoryFrequencyError( frequency=data_frequency) # Currencies don't always start trading at midnight. # Checking the last minute of the day instead. range_start = period_start.replace(hour=23, minute=59) \ if data_frequency == 'minute' else period_start has_data = range_in_bundle(asset, range_start, period_end, reader) if not has_data: log.debug('adding period: {}'.format(label)) chunks.append( dict(asset=asset, period_start=period_start, period_end=period_end, period=label)) dt += timedelta(days=1) chunks.sort(key=lambda chunk: chunk['period_end']) return chunks