def verify_get_candles_df(cls,
assets,
candles,
end_fixed_dt,
expected_df,
check_next_candle=False):
# run on all the fields
for field in ['volume', 'open', 'close', 'high', 'low']:
field_dt = cls.get_specific_field_from_df(expected_df, field,
assets[0])
# run on several timestamps
for delta in range(5):
end_dt = end_fixed_dt + timedelta(minutes=delta)
assert (field_dt.equals(
get_candles_df({assets[0]: candles},
field,
'5T',
3,
end_dt=end_dt)))
field_dt_a1 = cls.get_specific_field_from_df(
expected_df, field, assets[0])
field_dt_a2 = cls.get_specific_field_from_df(
expected_df, field, assets[1])
observed_df = get_candles_df(
{
assets[0]: candles,
assets[1]: candles
},
field,
'5T',
3,
end_dt=end_dt)
assert (observed_df.equals(
concat([field_dt_a1, field_dt_a2], axis=1)))
if check_next_candle:
# one candle forward
end_dt = end_fixed_dt + timedelta(minutes=6)
observed_df = get_candles_df(
{
assets[0]: candles,
assets[1]: candles
},
field,
'5T',
3,
end_dt=end_dt)
assert (not observed_df.equals(
concat([field_dt_a1, field_dt_a2], axis=1)))
assert (concat([field_dt_a1, field_dt_a2],
axis=1)[1:].equals(observed_df[:-1]))
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
sample_minutes
Returns
-------
"""
data = dict()
log.info('creating data sample from bundle')
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
log.info('bundle data:\n{}'.format(data['bundle'].tail(10)))
log.info('creating data sample from exchange api')
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
assets=assets,
bar_count=bar_count,
)
data['exchange'] = get_candles_df(
candles=candles,
field='close',
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
log.info('exchange data:\n{}'.format(data['exchange'].tail(10)))
for source in data:
df = data[source]
path = output_df(df, assets, '{}_{}'.format(freq, source))
log.info('saved {}:\n{}'.format(source, path))
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
check_less_precise=True,
)
def get_history_window(self,
assets,
end_dt,
bar_count,
frequency,
field,
data_frequency=None,
is_current=False):
"""
Public API method that returns a dataframe containing the requested
history window. Data is fully adjusted.
Parameters
----------
assets : list[TradingPair]
The assets whose data is desired.
end_dt: datetime
The date of the last bar
bar_count: int
The number of bars desired.
frequency: string
"1d" or "1m"
field: string
The desired field of the asset.
data_frequency: string
The frequency of the data to query; i.e. whether the data is
'daily' or 'minute' bars.
is_current: bool
Skip date filters when current data is requested (last few bars
until now).
Notes
-----
Catalysts requires an end data with bar count both CCXT wants a
start data with bar count. Since we have to make calculations here,
we ensure that the last candle match the end_dt parameter.
Returns
-------
DataFrame
A dataframe containing the requested data.
"""
freq, candle_size, unit, data_frequency = get_frequency(
frequency, data_frequency, supported_freqs=['T', 'D', 'H'])
# we want to avoid receiving empty candles
# so we request more than needed
# TODO: consider defining a const per asset
# and/or some retry mechanism (in each iteration request more data)
min_candles_number = get_candles_number_from_minutes(
unit, candle_size, self.MIN_MINUTES_REQUESTED)
requested_bar_count = bar_count
if requested_bar_count < min_candles_number:
requested_bar_count = min_candles_number
# The get_history method supports multiple asset
candles = self.get_candles(
freq=freq,
assets=assets,
bar_count=requested_bar_count,
end_dt=end_dt if not is_current else None,
)
# candles sanity check - verify no empty candles were received:
for asset in candles:
if not candles[asset]:
raise NoCandlesReceivedFromExchange(
bar_count=requested_bar_count,
end_dt=end_dt,
asset=asset,
exchange=self.name)
# for avoiding unnecessary forward fill end_dt is taken back one second
forward_fill_till_dt = end_dt - timedelta(seconds=1)
series = get_candles_df(candles=candles,
field=field,
freq=frequency,
bar_count=requested_bar_count,
end_dt=forward_fill_till_dt)
# TODO: consider how to approach this edge case
# delta_candle_size = candle_size * 60 if unit == 'H' else candle_size
# Checking to make sure that the dates match
# delta = get_delta(delta_candle_size, data_frequency)
# adj_end_dt = end_dt - delta
# last_traded = asset_series.index[-1]
# if last_traded < adj_end_dt:
# raise LastCandleTooEarlyError(
# last_traded=last_traded,
# end_dt=adj_end_dt,
# exchange=self.name,
# )
df = pd.DataFrame(series)
df.dropna(inplace=True)
return df.tail(bar_count)
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing data for {}/{} with {} timeframe until {}'.format(
exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field='close',
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle first / last row:\n{}'.format(
data['bundle'].iloc[[-1, 0]]
)
)
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
assets=assets,
bar_count=bar_count,
)
data['exchange'] = get_candles_df(
candles=candles,
field='close',
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange first / last row:\n{}'.format(
data['exchange'].iloc[[-1, 0]]
)
)
for source in data:
df = data[source]
path, folder = output_df(
df, assets, '{}_{}'.format(freq, source)
)
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'],
left=data['exchange'],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print('Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e))
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
pass
def compare_bundle_with_exchange(self, exchange, assets, end_dt, bar_count,
freq, data_frequency, data_portal, field):
"""
Creates DataFrames from the bundle and exchange for the specified
data set.
Parameters
----------
exchange: Exchange
assets
end_dt
bar_count
freq
data_frequency
data_portal
Returns
-------
"""
data = dict()
log_catcher = TestHandler()
with log_catcher:
symbols = [asset.symbol for asset in assets]
print(
'comparing {} for {}/{} with {} timeframe until {}'.format(
field, exchange.name, symbols, freq, end_dt
)
)
data['bundle'] = data_portal.get_history_window(
assets=assets,
end_dt=end_dt,
bar_count=bar_count,
frequency=freq,
field=field,
data_frequency=data_frequency,
)
set_print_settings()
print(
'the bundle data:\n{}'.format(
data['bundle']
)
)
candles = exchange.get_candles(
end_dt=end_dt,
freq=freq,
assets=assets,
bar_count=bar_count,
)
data['exchange'] = get_candles_df(
candles=candles,
field=field,
freq=freq,
bar_count=bar_count,
end_dt=end_dt,
)
print(
'the exchange data:\n{}'.format(
data['exchange']
)
)
for source in data:
df = data[source]
path, folder = output_df(
df, assets, '{}_{}'.format(freq, source)
)
print('saved {} test results: {}'.format(end_dt, folder))
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=1,
)
try:
assert_frame_equal(
right=data['bundle'][:-1],
left=data['exchange'][:-1],
check_less_precise=min([a.decimals for a in assets]),
)
except Exception as e:
print(
'Some differences were found within a 1 decimal point '
'interval of confidence: {}'.format(e)
)
with open(os.path.join(folder, 'compare.txt'), 'w+') as handle:
handle.write(e.args[0])
pass