def test_fallback_to_cached_values_within_a_month(inquirer): # pylint: disable=unused-argument
def mock_api_remote_fail(url, timeout): # pylint: disable=unused-argument
return MockResponse(500, '{"msg": "shit hit the fan"')
# Get a date 15 days ago and insert a cached entry for EUR JPY then
# Get a date 31 days ago and insert a cache entry for EUR CNY then
now = ts_now()
eurjpy_val = Price(FVal('124.123'))
cache_data = [
HistoricalPrice(
from_asset=A_EUR,
to_asset=A_JPY,
source=HistoricalPriceOracle.XRATESCOM,
timestamp=Timestamp(now - 86400 * 15),
price=eurjpy_val,
),
HistoricalPrice(
from_asset=A_EUR,
to_asset=A_CNY,
source=HistoricalPriceOracle.XRATESCOM,
timestamp=Timestamp(now - 86400 * 31),
price=Price(FVal('7.719')),
)
]
GlobalDBHandler().add_historical_prices(cache_data)
with patch('requests.get', side_effect=mock_api_remote_fail):
# We fail to find a response but then go back 15 days and find the cached response
result = inquirer._query_fiat_pair(A_EUR, A_JPY)
assert result == eurjpy_val
# The cached response for EUR CNY is too old so we will fail here
with pytest.raises(RemoteError):
result = inquirer._query_fiat_pair(A_EUR, A_CNY)
def test_cryptocompare_histohour_data_going_backward(data_dir, database,
freezer):
"""Test that the cryptocompare histohour data retrieval works properly
This test checks that doing an additional query in the past workd properly
and that the cached data are properly appended to the cached result. In production
this scenario should not happen often. Only way to happen if cryptocompare somehow adds
older data than what was previously queried.
"""
globaldb = GlobalDBHandler()
# first timestamp cryptocompare has histohour BTC/USD when queried from this test is
btc_start_ts = 1279936800
# first timestamp cryptocompare has histohour BTC/USD is: 1279940400
now_ts = btc_start_ts + 3600 * 2000 + 122
# create a cache file for BTC_USD
cache_data = [
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_USD,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1301536800),
price=Price(FVal('0.298')),
),
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_USD,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1301540400),
price=Price(FVal('0.298')),
)
]
globaldb.add_historical_prices(cache_data)
freezer.move_to(datetime.fromtimestamp(now_ts))
cc = Cryptocompare(data_directory=data_dir, database=database)
cc.query_and_store_historical_data(
from_asset=A_BTC,
to_asset=A_USD,
timestamp=now_ts - 3600 * 2 - 55,
)
result = get_globaldb_cache_entries(from_asset=A_BTC, to_asset=A_USD)
assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2
check_cc_result(result, forward=False)
data_range = globaldb.get_historical_price_range(
A_BTC, A_USD, HistoricalPriceOracle.CRYPTOCOMPARE) # noqa: E501
assert data_range[0] == btc_start_ts
assert data_range[
1] == 1301540400 # that's the closest ts to now_ts cc returns
def get_historical_price(
from_asset: 'Asset',
to_asset: 'Asset',
timestamp: Timestamp,
max_seconds_distance: int,
source: Optional[HistoricalPriceOracle] = None,
) -> Optional['HistoricalPrice']:
"""Gets the price around a particular timestamp
If no price can be found returns None
"""
connection = GlobalDBHandler()._conn
cursor = connection.cursor()
querystr = (
'SELECT from_asset, to_asset, source_type, timestamp, price FROM price_history '
'WHERE from_asset=? AND to_asset=? AND ABS(timestamp - ?) <= ? ')
querylist = [
from_asset.identifier, to_asset.identifier, timestamp,
max_seconds_distance
]
if source is not None:
querystr += ' AND source_type=?'
querylist.append(source.serialize_for_db())
querystr += 'ORDER BY ABS(timestamp - ?) ASC LIMIT 1'
querylist.append(timestamp)
query = cursor.execute(querystr, tuple(querylist))
result = query.fetchone()
if result is None:
return None
return HistoricalPrice.deserialize_from_db(result)
def test_manual_oracle_correctly_returns_price(globaldb, fake_price_historian):
"""Test that the manual oracle correctly returns price for asset"""
price_historian = fake_price_historian
# Add price at timestamp
globaldb.add_single_historical_price(
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_USD,
price=30000,
timestamp=Timestamp(1611595470),
source=HistoricalPriceOracle.MANUAL,
), )
# Make the other oracles fail
expected_price = Price(FVal('30000'))
oracle_instances = price_historian._oracle_instances
oracle_instances[
1].query_historical_price.side_effect = PriceQueryUnsupportedAsset(
'bitcoin')
oracle_instances[
2].query_historical_price.side_effect = PriceQueryUnsupportedAsset(
'bitcoin')
# Query price, should return the manual price
price = price_historian.query_historical_price(
from_asset=A_BTC,
to_asset=A_USD,
timestamp=Timestamp(1611595466),
)
assert price == expected_price
# Try to get manual price for a timestamp not in db
with pytest.raises(NoPriceForGivenTimestamp):
price = price_historian.query_historical_price(
from_asset=A_BTC,
to_asset=A_USD,
timestamp=Timestamp(1610595466),
)
def test_price_queries(price_historian, data_dir, database):
"""Test some historical price queries. Make sure that we test some
assets not in cryptocompare but in coigecko so the backup mechanism triggers and works"""
# These should hit cryptocompare
assert price_historian.query_historical_price(A_BTC, A_EUR,
1479200704) == FVal('663.66')
assert price_historian.query_historical_price(
A_XMR, A_BTC, 1579200704) == FVal('0.007526')
# this should hit the cryptocompare cache we are creating here
cache_data = [
HistoricalPrice(
from_asset=A_DASH,
to_asset=A_USD,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1438387200),
price=Price(FVal('10')),
),
HistoricalPrice(
from_asset=A_DASH,
to_asset=A_USD,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1438390800),
price=Price(FVal('20')),
)
]
GlobalDBHandler().add_historical_prices(cache_data)
price_historian._PriceHistorian__instance._cryptocompare = Cryptocompare(
data_directory=data_dir,
database=database,
)
price_historian.set_oracles_order(price_historian._oracles)
assert price_historian.query_historical_price(A_DASH, A_USD,
1438387700) == FVal('10')
# this should hit coingecko, since cornichon is not in cryptocompare
expected_price = FVal('0.07830444726516915')
assert price_historian.query_historical_price(A_CORN, A_USD,
1608854400) == expected_price
def get_globaldb_cache_entries(from_asset: Asset, to_asset: Asset) -> List[HistoricalPrice]:
"""TODO: This should probaly be moved in the globaldb/handler.py if we use it elsewhere
and made more generic (accept different sources)"""
connection = GlobalDBHandler()._conn
cursor = connection.cursor()
query = cursor.execute(
'SELECT from_asset, to_asset, source_type, timestamp, price FROM '
'price_history WHERE from_asset=? AND to_asset=? AND source_type=? ORDER BY timestamp ASC',
(
from_asset.identifier,
to_asset.identifier,
HistoricalPriceOracle.CRYPTOCOMPARE.serialize_for_db(), # pylint: disable=no-member
),
)
return [HistoricalPrice.deserialize_from_db(x) for x in query]
def test_parsing_forex_cache_works(
inquirer,
data_dir,
mocked_current_prices,
current_price_oracles_order,
): # pylint: disable=unused-argument
price = Price(FVal('124.123'))
now = ts_now()
cache_data = [HistoricalPrice(
from_asset=A_EUR,
to_asset=A_JPY,
source=HistoricalPriceOracle.XRATESCOM,
timestamp=Timestamp(now - 2000),
price=price,
)]
GlobalDBHandler().add_historical_prices(cache_data)
assert inquirer._query_fiat_pair(A_EUR, A_JPY) == price
def query_historical_fiat_exchange_rates(
from_fiat_currency: Asset,
to_fiat_currency: Asset,
timestamp: Timestamp,
) -> Optional[Price]:
assert from_fiat_currency.is_fiat(
), 'fiat currency should have been provided'
assert to_fiat_currency.is_fiat(
), 'fiat currency should have been provided'
# Check cache
price_cache_entry = GlobalDBHandler().get_historical_price(
from_asset=from_fiat_currency,
to_asset=to_fiat_currency,
timestamp=timestamp,
max_seconds_distance=DAY_IN_SECONDS,
)
if price_cache_entry:
return price_cache_entry.price
try:
prices_map = get_historical_xratescom_exchange_rates(
from_asset=from_fiat_currency,
time=timestamp,
)
except RemoteError:
return None
# Since xratecoms has daily rates let's save at timestamp of UTC day start
for asset, asset_price in prices_map.items():
GlobalDBHandler().add_historical_prices(entries=[
HistoricalPrice(
from_asset=from_fiat_currency,
to_asset=asset,
source=HistoricalPriceOracle.XRATESCOM,
timestamp=timestamp_to_daystart_timestamp(timestamp),
price=asset_price,
)
])
if asset == to_fiat_currency:
rate = asset_price
log.debug('Historical fiat exchange rate query succesful', rate=rate)
return rate
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> Price:
vs_currency = Coingecko.check_vs_currencies(
from_asset=from_asset,
to_asset=to_asset,
location='historical price',
)
if not vs_currency:
return Price(ZERO)
try:
from_coingecko_id = from_asset.to_coingecko()
except UnsupportedAsset:
log.warning(
f'Tried to query coingecko historical price from {from_asset.identifier} '
f'to {to_asset.identifier}. But from_asset is not supported in coingecko',
)
return Price(ZERO)
# check DB cache
price_cache_entry = GlobalDBHandler().get_historical_price(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
max_seconds_distance=DAY_IN_SECONDS,
source=HistoricalPriceOracle.COINGECKO,
)
if price_cache_entry:
return price_cache_entry.price
# no cache, query coingecko for daily price
date = timestamp_to_date(timestamp, formatstr='%d-%m-%Y')
result = self._query(
module='coins',
subpath=f'{from_coingecko_id}/history',
options={
'date': date,
'localization': False,
},
)
try:
price = Price(
FVal(result['market_data']['current_price'][vs_currency]))
except KeyError as e:
log.warning(
f'Queried coingecko historical price from {from_asset.identifier} '
f'to {to_asset.identifier}. But got key error for {str(e)} when '
f'processing the result.', )
return Price(ZERO)
# save result in the DB and return
date_timestamp = create_timestamp(date, formatstr='%d-%m-%Y')
GlobalDBHandler().add_historical_prices(entries=[
HistoricalPrice(
from_asset=from_asset,
to_asset=to_asset,
source=HistoricalPriceOracle.COINGECKO,
timestamp=date_timestamp,
price=price,
)
])
return price
def fixture_historical_price_test_data(globaldb):
data = [
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1428994442),
price=Price(FVal(210.865)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1439048640),
price=Price(FVal(1.13)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1511626623),
price=Price(FVal(396.56)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1511626622),
price=Price(FVal(394.56)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1539713117),
price=Price(FVal(178.615)),
),
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1539713117),
price=Price(FVal(5626.17)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481088),
price=Price(FVal(2044.76)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481095),
price=Price(FVal(2045.76)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481101),
price=Price(FVal(2049.76)),
),
HistoricalPrice(
from_asset=A_BTC,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481102),
price=Price(FVal(52342.5)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481103),
price=Price(FVal(2049.96)),
),
HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481196),
price=Price(FVal(2085.76)),
)
]
globaldb.add_historical_prices(data)
def _query_fiat_pair(base: Asset, quote: Asset) -> Price:
"""Queries the current price between two fiat assets
If a current price is not found but a cached price within 30 days is found
then that one is used.
May raise RemoteError if a price can not be found
"""
if base == quote:
return Price(FVal('1'))
now = ts_now()
# Check cache for a price within the last 24 hrs
price_cache_entry = GlobalDBHandler().get_historical_price(
from_asset=base,
to_asset=quote,
timestamp=now,
max_seconds_distance=DAY_IN_SECONDS,
)
if price_cache_entry:
return price_cache_entry.price
# Use the xratescom query and save all prices in the cache
price = None
try:
price_map = get_current_xratescom_exchange_rates(base)
for quote_asset, quote_price in price_map.items():
if quote_asset == quote:
# if the quote asset price is found return it
price = quote_price
GlobalDBHandler().add_historical_prices(entries=[
HistoricalPrice(
from_asset=base,
to_asset=quote_asset,
source=HistoricalPriceOracle.XRATESCOM,
timestamp=timestamp_to_daystart_timestamp(now),
price=quote_price,
)
])
if price: # the quote asset may not be found
return price
except RemoteError:
pass # price remains None
# query backup api
price = _query_currency_converterapi(base, quote)
if price is not None:
return price
# Check cache
price_cache_entry = GlobalDBHandler().get_historical_price(
from_asset=base,
to_asset=quote,
timestamp=now,
max_seconds_distance=MONTH_IN_SECONDS,
)
if price_cache_entry:
log.debug(
f'Could not query online apis for a fiat price. '
f'Used cached value from '
f'{(now - price_cache_entry.timestamp) / DAY_IN_SECONDS} days ago.',
base_currency=base.identifier,
quote_currency=quote.identifier,
price=price_cache_entry.price,
)
return price_cache_entry.price
# else
raise RemoteError(
f'Could not find a current {base.identifier} price for {quote.identifier}',
)
def test_get_historical_price(globaldb, historical_price_test_data): # pylint: disable=unused-argument # noqa: E501
# test normal operation, multiple arguments
expected_entry = HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(1511626623),
price=Price(FVal(396.56)),
)
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_EUR,
timestamp=1511627623,
max_seconds_distance=3600,
)
assert expected_entry == price_entry
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_EUR,
timestamp=1511627623,
max_seconds_distance=3600,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
)
assert expected_entry == price_entry
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_EUR,
timestamp=1511627623,
max_seconds_distance=3600,
source=HistoricalPriceOracle.MANUAL,
)
assert price_entry is None
# nothing in a small distance
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_EUR,
timestamp=1511627623,
max_seconds_distance=10,
)
assert price_entry is None
# multiple possible entries, make sure closest is returned
expected_entry = HistoricalPrice(
from_asset=A_ETH,
to_asset=A_EUR,
source=HistoricalPriceOracle.COINGECKO,
timestamp=Timestamp(1618481101),
price=Price(FVal(2049.76)),
)
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_EUR,
timestamp=1618481099,
max_seconds_distance=3600,
)
assert expected_entry == price_entry
# missing from asset
price_entry = globaldb.get_historical_price(
from_asset=A_BAL,
to_asset=A_EUR,
timestamp=1618481099,
max_seconds_distance=3600,
)
assert price_entry is None
# missing to asset
price_entry = globaldb.get_historical_price(
from_asset=A_ETH,
to_asset=A_USD,
timestamp=1618481099,
max_seconds_distance=3600,
)
assert price_entry is None
def query_and_store_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> None:
"""
Get historical hour price data from cryptocompare and populate the global DB
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical hour price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
now_ts = ts_now()
# save time at start of the query, in case the query does not complete due to rate limit
self.last_histohour_query_ts = now_ts
range_result = GlobalDBHandler().get_historical_price_range(
from_asset=from_asset,
to_asset=to_asset,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
)
if range_result is not None:
first_cached_ts, last_cached_ts = range_result
if timestamp > last_cached_ts:
# We have a cache but the requested timestamp does not hit it
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=last_cached_ts,
)
else:
# only other possibility, timestamp < cached start_time
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=first_cached_ts,
to_timestamp=Timestamp(0),
)
else:
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=Timestamp(0),
)
calculated_history = list(new_data)
if len(calculated_history) == 0:
return
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# Turn them into the format we will enter in the DB
prices = []
for entry in calculated_history:
try:
price = Price(
(deserialize_price(entry['high']) +
deserialize_price(entry['low'])) / 2) # noqa: E501
if price == Price(ZERO):
continue # don't write zero prices
prices.append(
HistoricalPrice(
from_asset=from_asset,
to_asset=to_asset,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
timestamp=Timestamp(entry['time']),
price=price,
))
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
log.error(
f'{msg}. Error getting price entry from cryptocompare histohour '
f'price results. Skipping entry.', )
continue
GlobalDBHandler().add_historical_prices(prices)
self.last_histohour_query_ts = ts_now(
) # also save when last query finished