def get_exchange_history_window(self,
exchange_name,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency,
ffill=True):
"""
Fetching price history window from the exchange bundle.
Parameters
----------
exchange: Exchange
assets: list[TradingPair]
end_dt: datetime
bar_count: int
frequency: str
field: str
data_frequency: str
ffill: bool
Returns
-------
DataFrame
"""
# TODO: verify that the exchange supports the timeframe
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
freq, candle_size, unit, adj_data_frequency = get_frequency(
frequency, data_frequency)
adj_bar_count = candle_size * bar_count
trailing_bar_count = candle_size - 1
if data_frequency == 'minute' and adj_data_frequency == 'daily':
end_dt = end_dt.floor('1D')
series = bundle.get_history_window_series_and_load(
assets=assets,
end_dt=end_dt,
bar_count=adj_bar_count,
field=field,
data_frequency=adj_data_frequency,
algo_end_dt=self._last_available_session,
trailing_bar_count=trailing_bar_count,
)
df = resample_history_df(pd.DataFrame(series), freq, field)
return df
def get_history_window_with_bundle(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
ffill=True,
force_auto_ingest=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar.
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
# TODO: fill how?
ffill: boolean
Forward-fill missing values. Only has effect if field
is 'price'.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
# TODO: this function needs some work,
# we're currently using it just for benchmark data
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D'])
adj_bar_count = candle_size * bar_count
try:
series = self.bundle.get_history_window_series_and_load(
assets=assets,
end_dt=end_dt,
bar_count=adj_bar_count,
field=field,
data_frequency=data_frequency,
force_auto_ingest=force_auto_ingest)
except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
series = dict()
for asset in assets:
if asset not in series or series[asset].index[-1] < end_dt:
# Adding bars too recent to be contained in the consolidated
# exchanges bundles. We go directly against the exchange
# to retrieve the candles.
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
trailing_dt = \
series[asset].index[-1] + get_delta(1, data_frequency) \
if asset in series else start_dt
# The get_history method supports multiple asset
# Use the original frequency to let each api optimize
# the size of result sets
trailing_bars = get_periods(trailing_dt, end_dt, freq)
candles = self.get_candles(
freq=freq,
assets=asset,
end_dt=end_dt,
bar_count=trailing_bars if trailing_bars < 500 else 500,
)
last_value = series[asset].iloc(0) if asset in series \
else np.nan
# Create a series with the common data_frequency, ffill
# missing values
candle_series = self.get_series_from_candles(
candles=candles,
start_dt=trailing_dt,
end_dt=end_dt,
data_frequency=data_frequency,
field=field,
previous_value=last_value)
if asset in series:
series[asset].append(candle_series)
else:
series[asset] = candle_series
df = resample_history_df(pd.DataFrame(series), freq, field)
# TODO: consider this more carefully
df.dropna(inplace=True)
return df
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H'])
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
min_candles_number = get_candles_number_from_minutes(
unit, candle_size, self.MIN_MINUTES_REQUESTED)
requested_bar_count = bar_count
if requested_bar_count < min_candles_number:
requested_bar_count = min_candles_number
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=requested_bar_count,
end_dt=end_dt if not is_current else None,
)
# candles sanity check - verify no empty candles were received:
for asset in candles:
if not candles[asset]:
raise NoCandlesReceivedFromExchange(
bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
# for avoiding unnecessary forward fill end_dt is taken back one second
forward_fill_till_dt = end_dt - timedelta(seconds=1)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df.tail(bar_count)
def get_exchange_history_window(self,
exchange_name,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency,
ffill=True):
"""
Fetching price history window from the exchange bundle.
Parameters
----------
exchange: Exchange
assets: list[TradingPair]
end_dt: datetime
bar_count: int
frequency: str
field: str
data_frequency: str
ffill: bool
Returns
-------
DataFrame
"""
# TODO: verify that the exchange supports the timeframe
bundle = self.exchange_bundles[exchange_name] # type: ExchangeBundle
freq, candle_size, unit, adj_data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D'])
adj_bar_count = candle_size * bar_count
if data_frequency == "minute":
# for minute frequency always request data until the
# current minute (do not include the current minute)
last_dt_for_series = end_dt - datetime.timedelta(minutes=1)
# read the minute bundles for daily frequency to
# support last partial candle
# TODO: optimize this by applying this logic only for the last day
if adj_data_frequency == 'daily':
adj_data_frequency = 'minute'
adj_bar_count = adj_bar_count * 1440
else: # data_frequency == "daily":
last_dt_for_series = end_dt
series = bundle.get_history_window_series_and_load(
assets=assets,
end_dt=last_dt_for_series,
bar_count=adj_bar_count,
field=field,
data_frequency=adj_data_frequency,
algo_end_dt=self._last_available_session,
)
start_dt = get_start_dt(last_dt_for_series, adj_bar_count,
adj_data_frequency, False)
df = resample_history_df(pd.DataFrame(series), freq, field, start_dt)
return df
def get_history_window_with_bundle(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
ffill=True,
force_auto_ingest=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar.
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
# TODO: fill how?
ffill: boolean
Forward-fill missing values. Only has effect if field
is 'price'.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
# TODO: this function needs some work, we're currently using it just for benchmark data
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency
)
adj_bar_count = candle_size * bar_count
try:
series = self.bundle.get_history_window_series_and_load(
assets=assets,
end_dt=end_dt,
bar_count=adj_bar_count,
field=field,
data_frequency=data_frequency,
force_auto_ingest=force_auto_ingest
)
except (PricingDataNotLoadedError, NoDataAvailableOnExchange):
series = dict()
for asset in assets:
if asset not in series or series[asset].index[-1] < end_dt:
# Adding bars too recent to be contained in the consolidated
# exchanges bundles. We go directly against the exchange
# to retrieve the candles.
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
trailing_dt = \
series[asset].index[-1] + get_delta(1, data_frequency) \
if asset in series else start_dt
# The get_history method supports multiple asset
# Use the original frequency to let each api optimize
# the size of result sets
trailing_bars = get_periods(
trailing_dt, end_dt, freq
)
candles = self.get_candles(
freq=freq,
assets=asset,
end_dt=end_dt,
bar_count=trailing_bars if trailing_bars < 500 else 500,
)
last_value = series[asset].iloc(0) if asset in series \
else np.nan
# Create a series with the common data_frequency, ffill
# missing values
candle_series = self.get_series_from_candles(
candles=candles,
start_dt=trailing_dt,
end_dt=end_dt,
data_frequency=data_frequency,
field=field,
previous_value=last_value
)
if asset in series:
series[asset].append(candle_series)
else:
series[asset] = candle_series
df = resample_history_df(pd.DataFrame(series), freq, field)
# TODO: consider this more carefully
df.dropna(inplace=True)
return df
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency
)
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=bar_count,
end_dt=end_dt if not is_current else None,
)
series = dict()
for asset in candles:
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
# Checking to make sure that the dates match
delta = get_delta(candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
series[asset] = asset_series
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency)
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=bar_count,
end_dt=end_dt if not is_current else None,
)
series = dict()
for asset in candles:
first_candle = candles[asset][0]
asset_series = self.get_series_from_candles(
candles=candles[asset],
start_dt=first_candle['last_traded'],
end_dt=end_dt,
data_frequency=frequency,
field=field,
)
# Checking to make sure that the dates match
delta = get_delta(candle_size, data_frequency)
adj_end_dt = end_dt - delta
last_traded = asset_series.index[-1]
if last_traded < adj_end_dt:
raise LastCandleTooEarlyError(
last_traded=last_traded,
end_dt=adj_end_dt,
exchange=self.name,
)
series[asset] = asset_series
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df
def get_history_window_for_asset(self,
asset,
end_dt,
bar_count,
frequency,
fields,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
asset : TradingPair
The asset whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
fields: string
The desired fields of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H'])
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
min_candles_number = get_candles_number_from_minutes(
unit, candle_size, self.MIN_MINUTES_REQUESTED)
requested_bar_count = bar_count
if requested_bar_count < min_candles_number:
requested_bar_count = min_candles_number
candles = self.get_candles(
freq=freq,
assets=
asset, # when we pass a single asset, the response is a flat DataFrame with the fields as columns
bar_count=requested_bar_count,
end_dt=end_dt if not is_current else None,
)
# candles sanity check - verify no empty candles were received:
if len(candles) == 0:
raise NoCandlesReceivedFromExchange(bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
# for avoiding unnecessary forward fill end_dt is taken back one second
forward_fill_till_dt = end_dt - timedelta(seconds=1)
candles_df = get_candles_df_for_asset(candles=candles,
fields=fields,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
return candles_df.tail(bar_count)
