def map_frequency(value, source='ccxt', raise_error=True):
"""
Map a frequency value between CCXT and Catalyst
Parameters
----------
value: str
source: str
raise_error: bool
Returns
-------
Notes
-----
The Pandas offset aliases supported by Catalyst:
Alias Description
W weekly frequency
M month end frequency
D calendar day frequency
H hourly frequency
T, min minutely frequency
The CCXT timeframes:
'1m': '1minute',
'1h': '1hour',
'1d': '1day',
'1w': '1week',
'1M': '1month',
'1y': '1year',
"""
match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T|w|W|min)', value,
re.M | re.I)
if match:
candle_size = int(match.group(1)) \
if match.group(1) else 1
unit = match.group(2)
else:
raise ValueError('Unable to parse frequency or timeframe')
if source == 'ccxt':
if unit == 'd':
result = '{}D'.format(candle_size)
elif unit == 'm':
result = '{}T'.format(candle_size)
elif unit == 'h':
result = '{}H'.format(candle_size)
elif unit == 'w':
result = '{}W'.format(candle_size)
elif unit == 'M':
result = '{}M'.format(candle_size)
elif raise_error:
raise InvalidHistoryTimeframeError(timeframe=value)
else:
if unit == 'D':
result = '{}d'.format(candle_size)
elif unit == 'min' or unit == 'T':
result = '{}m'.format(candle_size)
elif unit == 'H':
result = '{}h'.format(candle_size)
elif unit == 'W':
result = '{}w'.format(candle_size)
elif unit == 'M':
result = '{}M'.format(candle_size)
elif raise_error:
raise InvalidHistoryFrequencyError(frequency=value)
return result
def get_candles(self,
freq,
assets,
bar_count=None,
start_dt=None,
end_dt=None):
"""
Retrieve OHLVC candles from Poloniex
:param freq:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'5m', '15m', '30m', '2h', '4h', '1D'
"""
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
log.debug('retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)))
if freq == '1T' and (bar_count == 1 or bar_count is None):
# TODO: use the order book instead
# We use the 5m to fetch the last bar
frequency = 300
elif freq == '5T':
frequency = 300
elif freq == '15T':
frequency = 900
elif freq == '30T':
frequency = 1800
elif freq == '120T':
frequency = 7200
elif freq == '240T':
frequency = 14400
elif freq == '1D':
frequency = 86400
else:
# Poloniex does not offer 1m data candles
# It is likely to error out there frequently
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
delta = end_dt - pd.to_datetime('1970-1-1', utc=True)
end = int(delta.total_seconds())
if bar_count is None:
start = end - 2 * frequency
else:
start = end - bar_count * frequency
try:
response = self.api.returnchartdata(self.get_symbol(asset),
frequency, start, end)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content))
def ohlc_from_candle(candle):
last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
last_traded = last_traded.replace(tzinfo=pytz.UTC)
ohlc = dict(open=np.float64(candle['open']),
high=np.float64(candle['high']),
low=np.float64(candle['low']),
close=np.float64(candle['close']),
volume=np.float64(candle['volume']),
price=np.float64(candle['close']),
last_traded=last_traded)
return ohlc
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(response[0])
else:
ohlc_bars = []
for candle in response:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_frequency(freq, data_frequency=None, supported_freqs=['D', 'H', 'T']):
"""
Takes an arbitrary candle size (e.g. 15T) and converts to the lowest
common denominator supported by the data bundles (e.g. 1T). The data
bundles only support 1T and 1D frequencies. If another frequency
is requested, Catalyst must request the underlying data and resample.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if data_frequency is None:
data_frequency = 'daily' if freq.upper().endswith('D') else 'minute'
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# TODO: some exchanges support H and W frequencies but not bundles
# Find a way to pass-through these parameters to exchanges
# but resample from minute or daily in backtest mode
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
if unit.lower() == 'd':
unit = 'D'
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
unit = 'T'
alias = '{}T'.format(candle_size)
data_frequency = 'minute'
elif unit.lower() == 'h':
data_frequency = 'minute'
if 'H' in supported_freqs:
unit = 'H'
alias = '{}H'.format(candle_size)
else:
candle_size = candle_size * 60
alias = '{}T'.format(candle_size)
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def get_candles(self,
data_frequency,
assets,
bar_count=None,
start_dt=None,
end_dt=None):
"""
Retrieve OHLVC candles from Poloniex
:param data_frequency:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'5m', '15m', '30m', '2h', '4h', '1D'
"""
# TODO: implement end_dt and start_dt filters
if (data_frequency == '5m' or data_frequency
== 'minute'): # TODO: Polo does not have '1m'
frequency = 300
elif (data_frequency == '15m'):
frequency = 900
elif (data_frequency == '30m'):
frequency = 1800
elif (data_frequency == '2h'):
frequency = 7200
elif (data_frequency == '4h'):
frequency = 14400
elif (data_frequency == '1D' or data_frequency == 'daily'):
frequency = 86400
else:
raise InvalidHistoryFrequencyError(frequency=data_frequency)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
end = int(time.time())
if (bar_count is None):
start = end - 2 * frequency
else:
start = end - bar_count * frequency
try:
response = self.api.returnchartdata(self.get_symbol(asset),
frequency, start, end)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content))
def ohlc_from_candle(candle):
last_traded = pd.Timestamp.utcfromtimestamp(candle['date'])
last_traded = last_traded.replace(tzinfo=pytz.UTC)
ohlc = dict(open=np.float64(candle['open']),
high=np.float64(candle['high']),
low=np.float64(candle['low']),
close=np.float64(candle['close']),
volume=np.float64(candle['volume']),
price=np.float64(candle['close']),
last_traded=last_traded)
return ohlc
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(response[0])
else:
ohlc_bars = []
for candle in response:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self, data_frequency, assets, bar_count=None):
"""
Retrieve OHLVC candles from Bitfinex
:param data_frequency:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
'1M'
"""
# TODO: use BcolzMinuteBarReader to read from cache
freq_match = re.match(r'([0-9].*)(m|h|d)', data_frequency, re.M | re.I)
if freq_match:
number = int(freq_match.group(1))
unit = freq_match.group(2)
if unit == 'd':
converted_unit = 'D'
else:
converted_unit = unit
frequency = '{}{}'.format(number, converted_unit)
allowed_frequencies = [
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D',
'14D', '1M'
]
if frequency not in allowed_frequencies:
raise InvalidHistoryFrequencyError(frequency=data_frequency)
elif data_frequency == 'minute':
frequency = '1m'
elif data_frequency == 'daily':
frequency = '1D'
else:
raise InvalidHistoryFrequencyError(frequency=data_frequency)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
symbol = self._get_v2_symbol(asset)
url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
url=self.url, frequency=frequency, symbol=symbol)
if bar_count:
is_list = True
url += '/hist?limit={}'.format(int(bar_count))
else:
is_list = False
url += '/last'
try:
response = requests.get(url)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response.content:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content))
candles = response.json()
def ohlc_from_candle(candle):
ohlc = dict(open=np.float64(candle[1]),
high=np.float64(candle[3]),
low=np.float64(candle[4]),
close=np.float64(candle[2]),
volume=np.float64(candle[5]),
price=np.float64(candle[2]),
last_traded=pd.Timestamp.utcfromtimestamp(
candle[0] / 1000.0))
return ohlc
if is_list:
ohlc_bars = []
# We can to list candles from old to new
for candle in reversed(candles):
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
else:
ohlc = ohlc_from_candle(candles)
ohlc_map[asset] = ohlc
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_frequency(freq, data_frequency):
"""
Get the frequency parameters.
Notes
-----
We're trying to use Pandas convention for frequency aliases.
Parameters
----------
freq: str
data_frequency: str
Returns
-------
str, int, str, str
"""
if freq == 'minute':
unit = 'T'
candle_size = 1
elif freq == 'daily':
unit = 'D'
candle_size = 1
else:
freq_match = re.match(r'([0-9].*)?(m|M|d|D|h|H|T)', freq, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1)) if freq_match.group(1) \
else 1
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# TODO: some exchanges support H and W frequencies but not bundles
# Find a way to pass-through these parameters to exchanges
# but resample from minute or daily in backtest mode
# see catalyst/exchange/ccxt/ccxt_exchange.py:242 for mapping between
# Pandas offet aliases (used by Catalyst) and the CCXT timeframes
if unit.lower() == 'd':
alias = '{}D'.format(candle_size)
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm' or unit == 'T':
alias = '{}T'.format(candle_size)
if data_frequency == 'daily':
data_frequency = 'minute'
# elif unit.lower() == 'h':
# candle_size = candle_size * 60
#
# alias = '{}T'.format(candle_size)
# if data_frequency == 'daily':
# data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyAlias(freq=freq)
return alias, candle_size, unit, data_frequency
def get_candles(self,
freq,
assets,
bar_count=None,
start_dt=None,
end_dt=None):
"""
Supported Intervals
-------------------
day, oneMin, fiveMin, thirtyMin, hour
:param freq:
:param assets:
:param bar_count:
:param start_dt
:param end_dt
:return:
"""
# TODO: this has no effect at the moment
if end_dt is None:
end_dt = pd.Timestamp.utcnow()
log.debug('retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)))
if freq == '1T':
frequency = 'oneMin'
elif freq == '5T':
frequency = 'fiveMin'
elif freq == '30T':
frequency = 'thirtyMin'
elif freq == '60T':
frequency = 'hour'
elif freq == '1D':
frequency = 'day'
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
for asset in asset_list:
end = int(time.mktime(end_dt.timetuple()))
url = '{url}/pub/market/GetTicks?marketName={symbol}' \
'&tickInterval={frequency}&_={end}'.format(
url=URL2,
symbol=self.get_symbol(asset),
frequency=frequency,
end=end, )
try:
data = json.loads(urllib.request.urlopen(url).read().decode())
except Exception as e:
raise ExchangeRequestError(error=e)
if data['message']:
raise ExchangeRequestError(
error='Unable to fetch candles {}'.format(data['message']))
candles = data['result']
def ohlc_from_candle(candle):
ohlc = dict(open=candle['O'],
high=candle['H'],
low=candle['L'],
close=candle['C'],
volume=candle['V'],
price=candle['C'],
last_traded=pd.to_datetime(candle['T'], utc=True))
return ohlc
ordered_candles = list(reversed(candles))
ohlc_map = dict()
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
else:
# TODO: optimize
ohlc_bars = []
for candle in ordered_candles[:bar_count]:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self,
freq,
assets,
bar_count=None,
start_dt=None,
end_dt=None):
"""
Retrieve OHLVC candles from Bitfinex
:param data_frequency:
:param assets:
:param bar_count:
:return:
Available Frequencies
---------------------
'1m', '5m', '15m', '30m', '1h', '3h', '6h', '12h', '1D', '7D', '14D',
'1M'
"""
log.debug('retrieving {bars} {freq} candles on {exchange} from '
'{end_dt} for markets {symbols}, '.format(
bars=bar_count,
freq=freq,
exchange=self.name,
end_dt=end_dt,
symbols=get_symbols_string(assets)))
allowed_frequencies = [
'1T', '5T', '15T', '30T', '60T', '180T', '360T', '720T', '1D',
'7D', '14D', '30D'
]
if freq not in allowed_frequencies:
raise InvalidHistoryFrequencyError(frequency=freq)
freq_match = re.match(r'([0-9].*)(T|H|D)', freq, re.M | re.I)
if freq_match:
number = int(freq_match.group(1))
unit = freq_match.group(2)
if unit == 'T':
if number in [60, 180, 360, 720]:
number = number / 60
converted_unit = 'h'
else:
converted_unit = 'm'
else:
converted_unit = unit
frequency = '{}{}'.format(number, converted_unit)
else:
raise InvalidHistoryFrequencyError(frequency=freq)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
symbol = self._get_v2_symbol(asset)
url = '{url}/v2/candles/trade:{frequency}:{symbol}'.format(
url=self.url, frequency=frequency, symbol=symbol)
if bar_count:
is_list = True
url += '/hist?limit={}'.format(int(bar_count))
def get_ms(date):
epoch = datetime.datetime.utcfromtimestamp(0)
epoch = epoch.replace(tzinfo=pytz.UTC)
return (date - epoch).total_seconds() * 1000.0
if start_dt is not None:
start_ms = get_ms(start_dt)
url += '&start={0:f}'.format(start_ms)
if end_dt is not None:
end_ms = get_ms(end_dt)
url += '&end={0:f}'.format(end_ms)
else:
is_list = False
url += '/last'
try:
self.ask_request()
response = requests.get(url)
except Exception as e:
raise ExchangeRequestError(error=e)
if 'error' in response.content:
raise ExchangeRequestError(
error='Unable to retrieve candles: {}'.format(
response.content))
candles = response.json()
def ohlc_from_candle(candle):
last_traded = pd.Timestamp.utcfromtimestamp(candle[0] / 1000.0)
last_traded = last_traded.replace(tzinfo=pytz.UTC)
ohlc = dict(open=np.float64(candle[1]),
high=np.float64(candle[3]),
low=np.float64(candle[4]),
close=np.float64(candle[2]),
volume=np.float64(candle[5]),
price=np.float64(candle[2]),
last_traded=last_traded)
return ohlc
if is_list:
ohlc_bars = []
# We can to list candles from old to new
for candle in reversed(candles):
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
else:
ohlc = ohlc_from_candle(candles)
ohlc_map[asset] = ohlc
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_candles(self,
data_frequency,
assets,
bar_count=None,
start_date=None):
"""
Supported Intervals
-------------------
day, oneMin, fiveMin, thirtyMin, hour
:param data_frequency:
:param assets:
:param bar_count:
:return:
"""
log.info('retrieving candles')
if data_frequency == 'minute' or data_frequency == '1m':
frequency = 'oneMin'
elif data_frequency == '5m':
frequency = 'fiveMin'
elif data_frequency == '30m':
frequency = 'thirtyMin'
elif data_frequency == '1h':
frequency = 'hour'
elif data_frequency == 'daily' or data_frequency == '1D':
frequency = 'day'
else:
raise InvalidHistoryFrequencyError(frequency=data_frequency)
# Making sure that assets are iterable
asset_list = [assets] if isinstance(assets, TradingPair) else assets
ohlc_map = dict()
for asset in asset_list:
url = '{url}/pub/market/GetTicks?marketName={symbol}' \
'&tickInterval={frequency}&_=1499127220008'.format(
url=URL2,
symbol=self.get_symbol(asset),
frequency=frequency
)
try:
data = json.loads(urllib.request.urlopen(url).read().decode())
except Exception as e:
raise ExchangeRequestError(error=e)
if data['message']:
raise ExchangeRequestError(
error='Unable to fetch candles {}'.format(data['message']))
candles = data['result']
def ohlc_from_candle(candle):
ohlc = dict(open=candle['O'],
high=candle['H'],
low=candle['L'],
close=candle['C'],
volume=candle['V'],
price=candle['C'],
last_traded=pd.to_datetime(candle['T'], utc=True))
return ohlc
ordered_candles = list(reversed(candles))
if bar_count is None:
ohlc_map[asset] = ohlc_from_candle(ordered_candles[0])
else:
ohlc_bars = []
for candle in ordered_candles[:bar_count]:
ohlc = ohlc_from_candle(candle)
ohlc_bars.append(ohlc)
ohlc_map[asset] = ohlc_bars
return ohlc_map[assets] \
if isinstance(assets, TradingPair) else ohlc_map
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
ffill=True):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list of catalyst.data.Asset objects
The assets whose data is desired.
end_dt: not applicable to cryptocurrencies
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
# TODO: fill how?
ffill: boolean
Forward-fill missing values. Only has effect if field
is 'price'.
Returns
-------
A dataframe containing the requested data.
"""
freq_match = re.match(r'([0-9].*)(m|M|d|D)', frequency, re.M | re.I)
if freq_match:
candle_size = int(freq_match.group(1))
unit = freq_match.group(2)
else:
raise InvalidHistoryFrequencyError(frequency)
if unit.lower() == 'd':
if data_frequency == 'minute':
data_frequency = 'daily'
elif unit.lower() == 'm':
if data_frequency == 'daily':
data_frequency = 'minute'
else:
raise InvalidHistoryFrequencyError(frequency)
adj_bar_count = candle_size * bar_count
try:
series = self.bundle.get_history_window_series_and_load(
assets=assets,
end_dt=end_dt,
bar_count=adj_bar_count,
field=field,
data_frequency=data_frequency)
except PricingDataNotLoadedError:
series = dict()
for asset in assets:
if asset not in series or series[asset].index[-1] < end_dt:
# Adding bars too recent to be contained in the consolidated
# exchanges bundles. We go directly against the exchange
# to retrieve the candles.
start_dt = get_start_dt(end_dt, adj_bar_count, data_frequency)
trailing_dt = \
series[asset].index[-1] + get_delta(1, data_frequency) \
if asset in series else start_dt
trailing_bar_count = \
get_periods(trailing_dt, end_dt, data_frequency)
# The get_history method supports multiple asset
candles = self.get_candles(data_frequency=data_frequency,
assets=asset,
bar_count=trailing_bar_count,
end_dt=end_dt)
last_value = series[asset].iloc(0) if asset in series \
else np.nan
candle_series = self.get_series_from_candles(
candles=candles,
start_dt=trailing_dt,
end_dt=end_dt,
field=field,
previous_value=last_value)
if asset in series:
series[asset].append(candle_series)
else:
series[asset] = candle_series
df = pd.DataFrame(series)
if candle_size > 1:
if field == 'open':
agg = 'first'
elif field == 'high':
agg = 'max'
elif field == 'low':
agg = 'min'
elif field == 'close':
agg = 'last'
elif field == 'volume':
agg = 'sum'
else:
raise ValueError('Invalid field.')
df = df.resample('{}T'.format(candle_size)).agg(agg)
return df
def prepare_chunks(self, assets, data_frequency, start_dt, end_dt):
"""
Split a price data request into chunks corresponding to individual
bundles.
:param assets:
:param data_frequency:
:param start_dt:
:param end_dt:
:return:
"""
reader = self.get_reader(data_frequency)
chunks = []
for asset in assets:
try:
asset_start, asset_end = \
get_adj_dates(start_dt, end_dt, [asset], data_frequency)
except NoDataAvailableOnExchange:
continue
start_dt = max(start_dt, self.calendar.first_trading_session)
start_dt = max(start_dt, asset_start)
# Aligning start / end dates with the daily calendar
sessions = get_periods_range(start_dt, end_dt, data_frequency) \
if data_frequency == 'minute' \
else self.calendar.sessions_in_range(start_dt, end_dt)
if asset_start < sessions[0]:
asset_start = sessions[0]
if asset_end > sessions[-1]:
asset_end = sessions[-1]
chunk_labels = []
dt = sessions[0]
while dt <= sessions[-1]:
label = '{}-{:02d}'.format(dt.year, dt.month) \
if data_frequency == 'minute' else '{}'.format(dt.year)
if label not in chunk_labels:
chunk_labels.append(label)
# Adjusting the period dates to match the availability
# of the trading pair
if data_frequency == 'minute':
period_start, period_end = get_month_start_end(dt)
asset_start_month, _ = get_month_start_end(asset_start)
if asset_start_month == period_start \
and period_start < asset_start:
period_start = asset_start
_, asset_end_month = get_month_start_end(asset_end)
if asset_end_month == period_end \
and period_end > asset_end:
period_end = asset_end
elif data_frequency == 'daily':
period_start, period_end = get_year_start_end(dt)
asset_start_year, _ = get_year_start_end(asset_start)
if asset_start_year == period_start \
and period_start < asset_start:
period_start = asset_start
_, asset_end_year = get_year_start_end(asset_end)
if asset_end_year == period_end \
and period_end > asset_end:
period_end = asset_end
else:
raise InvalidHistoryFrequencyError(
frequency=data_frequency)
# Currencies don't always start trading at midnight.
# Checking the last minute of the day instead.
range_start = period_start.replace(hour=23, minute=59) \
if data_frequency == 'minute' else period_start
has_data = range_in_bundle(asset, range_start, period_end,
reader)
if not has_data:
log.debug('adding period: {}'.format(label))
chunks.append(
dict(asset=asset,
period_start=period_start,
period_end=period_end,
period=label))
dt += timedelta(days=1)
chunks.sort(key=lambda chunk: chunk['period_end'])
return chunks
