def get_frequency(freq, data_frequency):
"""
Get the frequency parameters.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
if unit.lower() == 'd':
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
alias = '{}T'.format(candle_size)
if data_frequency == 'daily':
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def get_frequency(freq, data_frequency):
"""
Get the frequency parameters.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# TODO: some exchanges support H and W frequencies but not bundles
# Find a way to pass-through these parameters to exchanges
# but resample from minute or daily in backtest mode
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
if unit.lower() == 'd':
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
alias = '{}T'.format(candle_size)
if data_frequency == 'daily':
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
"""
Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
common denominator supported by the data bundles (e.g. 1T). The data
bundles only support 1T and 1D frequencies. If another frequency
is requested, Catalyst must request the underlying data and resample.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if data_frequency is None:
data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# TODO: some exchanges support H and W frequencies but not bundles
# Find a way to pass-through these parameters to exchanges
# but resample from minute or daily in backtest mode
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
if unit.lower() == 'd':
unit = 'D'
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
unit = 'T'
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
elif unit.lower() == 'h':
data_frequency = 'minute'
if 'H' in supported_freqs:
unit = 'H'
alias = '{}H'.format(candle_size)
else:
candle_size = candle_size * 60
alias = '{}T'.format(candle_size)
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H'])
if unit == 'H':
raise InvalidHistoryFrequencyAlias(freq=frequency)
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
kExtra_minutes_candles = 150
requested_bar_count = bar_count + \
get_candles_number_from_minutes(unit,
candle_size,
kExtra_minutes_candles)
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=requested_bar_count,
end_dt=end_dt if not is_current else None,
)
# candles sanity check - verify no empty candles were received:
for asset in candles:
if not candles[asset]:
raise NoCandlesReceivedFromExchange(
bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
# for avoiding unnecessary forward fill end_dt is taken back one second
forward_fill_till_dt = end_dt - timedelta(seconds=1)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df.tail(bar_count)