def test_price_queries(price_historian, data_dir, database):
"""Test some historical price queries. Make sure that we test some
assets not in cryptocompare but in coigecko so the backup mechanism triggers and works"""
# These should hit cryptocompare
assert price_historian.query_historical_price(A_BTC, A_EUR,
1479200704) == FVal('663.66')
assert price_historian.query_historical_price(
A_XMR, A_BTC, 1579200704) == FVal('0.007526')
# this should hit the cryptocompare cache we are creating here
contents = """{"start_time": 0, "end_time": 1439390800,
"data": [{"time": 1438387200, "close": 10, "high": 10, "low": 10, "open": 10,
"volumefrom": 10, "volumeto": 10}, {"time": 1438390800, "close": 20, "high": 20,
"low": 20, "open": 20, "volumefrom": 20, "volumeto": 20}]}"""
price_history_dir = get_or_make_price_history_dir(data_dir)
with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}DASH_USD.json',
'w') as f:
f.write(contents)
price_historian._PriceHistorian__instance._cryptocompare = Cryptocompare(
data_directory=data_dir,
database=database,
)
price_historian.set_oracles_order(price_historian._oracles)
assert price_historian.query_historical_price(A_DASH, A_USD,
1438387700) == FVal('10')
# this should hit coingecko, since cornichon is not in cryptocompare
cornichon = Asset('CORN-2')
expected_price = FVal('0.07830444726516915')
assert price_historian.query_historical_price(cornichon, A_USD,
1608854400) == expected_price
def __new__(
cls,
data_dir: Path = None,
cryptocompare: 'Cryptocompare' = None,
coingecko: 'Coingecko' = None,
) -> 'Inquirer':
if Inquirer.__instance is not None:
return Inquirer.__instance
assert data_dir, 'arguments should be given at the first instantiation'
assert cryptocompare, 'arguments should be given at the first instantiation'
assert coingecko, 'arguments should be given at the first instantiation'
Inquirer.__instance = object.__new__(cls)
Inquirer.__instance._data_directory = data_dir
Inquirer._cryptocompare = cryptocompare
Inquirer._coingecko = coingecko
Inquirer._cached_current_price = {}
# Make price history directory if it does not exist
price_history_dir = get_or_make_price_history_dir(data_dir)
filename = price_history_dir / 'price_history_forex.json'
try:
with open(filename, 'r') as f:
# we know price_history_forex contains a dict
data = rlk_jsonloads_dict(f.read())
Inquirer.__instance._cached_forex_data = data
except (OSError, JSONDecodeError):
Inquirer.__instance._cached_forex_data = {}
return Inquirer.__instance
def save_historical_forex_data() -> None:
instance = Inquirer()
# Make price history directory if it does not exist
price_history_dir = get_or_make_price_history_dir(instance._data_directory)
filename = price_history_dir / 'price_history_forex.json'
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(instance._cached_forex_data))
def test_cryptocompare_histohour_data_going_backward(data_dir, database, freezer):
"""Test that the cryptocompare histohour data retrieval works properly
This test checks that doing an additional query in the past workd properly
and that the cached data are properly appended to the cached result. In production
this scenario should not happen often. Only way to happen if cryptocompare somehow adds
older data than what was previously queried.
"""
# first timestamp cryptocompare has histohour BTC/USD when queried from this test is
btc_start_ts = 1279936800
# first timestamp cryptocompare has histohour BTC/USD is: 1279940400
now_ts = btc_start_ts + 3600 * 2000 + 122
# create a cache file for BTC_USD
contents = """{"start_time": 1301536800, "end_time": 1301540400,
"data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298,
"volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298,
"low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}"""
price_history_dir = get_or_make_price_history_dir(data_dir)
with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f:
f.write(contents)
freezer.move_to(datetime.fromtimestamp(now_ts))
cc = Cryptocompare(data_directory=data_dir, database=database)
result = cc.get_historical_data(
from_asset=A_BTC,
to_asset=A_USD,
timestamp=now_ts - 3600 * 2 - 55,
only_check_cache=False,
)
cache_key = PairCacheKey('BTC_USD')
assert len(result) == CRYPTOCOMPARE_HOURQUERYLIMIT * 3 + 2
check_cc_result(result, forward=False)
assert cache_key in cc.price_history
assert cc.price_history[cache_key].start_time == btc_start_ts
assert cc.price_history[cache_key].end_time == now_ts
check_cc_result(cc.price_history[cache_key].data, forward=False)
def _save_cached_price(
self,
from_asset: Asset,
to_asset: Asset,
date: str,
price: Price,
) -> None:
price_history_dir = get_or_make_price_history_dir(self.data_directory)
filename = (
price_history_dir /
f'{PRICE_HISTORY_FILE_PREFIX }{from_asset.identifier}_{to_asset.identifier}.json'
)
data: Dict[str, Price] = {}
if filename.is_file():
with open(filename, 'r') as f:
try:
data = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
data = {}
if not isinstance(data, dict):
data = {}
data[date] = price
with open(filename, 'w') as outfile:
outfile.write(rlk_jsondumps(data))
def test_cryptocompare_historical_data_use_cached_price(data_dir, database):
"""Test that the cryptocompare cache is used and also properly deserialized"""
# Create a cache file for SNGLS_BTC
contents = """{"start_time": 0, "end_time": 1439390800,
"data": [{"time": 1438387200, "close": 10, "high": 10, "low": 10, "open": 10,
"volumefrom": 10, "volumeto": 10}, {"time": 1438390800, "close": 20, "high": 20,
"low": 20, "open": 20, "volumefrom": 20, "volumeto": 20}]}"""
price_history_dir = get_or_make_price_history_dir(data_dir)
with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}SNGLS_BTC.json', 'w') as f:
f.write(contents)
cc = Cryptocompare(data_directory=data_dir, database=database)
with patch.object(cc, 'query_endpoint_histohour') as histohour_mock:
result = cc.get_historical_data(
from_asset=A_SNGLS,
to_asset=A_BTC,
timestamp=1438390801,
only_check_cache=False,
)
# make sure that histohour was not called, in essence that the cache was used
assert histohour_mock.call_count == 0
assert len(result) == 2
assert isinstance(result[0].low, FVal)
assert result[0].low == FVal(10)
assert isinstance(result[0].high, FVal)
assert result[0].high == FVal(10)
assert isinstance(result[1].low, FVal)
assert result[1].low == FVal(20)
assert isinstance(result[1].high, FVal)
assert result[1].high == FVal(20)
def __init__(self, data_directory: Path,
database: Optional[DBHandler]) -> None:
super().__init__(database=database,
service_name=ExternalService.CRYPTOCOMPARE)
self.data_directory = data_directory
self.price_history: Dict[PairCacheKey, PriceHistoryData] = {}
self.price_history_file: Dict[PairCacheKey, Path] = {}
self.session = requests.session()
self.session.headers.update({'User-Agent': 'rotkehlchen'})
self.last_histohour_query_ts = 0
self.last_rate_limit = 0
price_history_dir = get_or_make_price_history_dir(data_directory)
# Check the data folder and remember the filenames of any cached history
prefix = os.path.join(str(price_history_dir),
PRICE_HISTORY_FILE_PREFIX)
prefix = prefix.replace('\\', '\\\\')
regex = re.compile(prefix + r'(.*)\.json')
for file_ in price_history_dir.rglob(PRICE_HISTORY_FILE_PREFIX +
'*.json'):
file_str = str(file_).replace('\\\\', '\\')
match = regex.match(file_str)
assert match
cache_key = PairCacheKey(match.group(1))
self.price_history_file[cache_key] = file_
def upgrade_v22_to_v23(db: 'DBHandler') -> None:
"""Upgrades the DB from v22 to v23
- Migrates the settings entries 'thousand_separator', 'decimal_separator'
and 'currency_location' into the 'frontend_settings' entry.
- Deletes Bitfinex trades and their used query range, so trades can be
populated again with the right `fee_asset`.
- Delete all cryptocompare price cache files. Move forex price cache to price_history directory
"""
settings = ('"thousand_separator"', '"decimal_separator"', '"currency_location"')
cursor = db.conn.cursor()
# Get the settings and put them in a dict
setting_value_map = dict(
cursor.execute(
f'SELECT name, value FROM settings WHERE name IN ({",".join(settings)});',
).fetchall(),
)
# If the settings exist, migrate them into the 'frontend_settings' entry
if setting_value_map:
frontend_settings = cursor.execute(
'SELECT value FROM settings WHERE name = "frontend_settings";',
).fetchone()
if frontend_settings is not None:
setting_value_map.update(json.loads(frontend_settings[0]))
cursor.execute(
'INSERT OR REPLACE INTO settings(name, value) VALUES(?, ?)',
('frontend_settings', json.dumps(setting_value_map)),
)
# Delete the settings
cursor.execute(f'DELETE FROM settings WHERE name IN ({",".join(settings)});')
# Delete Bitfinex used_query_ranges
cursor.execute('DELETE FROM used_query_ranges WHERE name = "bitfinex_trades";')
# Delete Bitfinex trades
cursor.execute('DELETE FROM trades WHERE location = "T";')
# Delete deprecated historical data start setting
cursor.execute('DELETE from settings WHERE name="historical_data_start";')
db.conn.commit()
# -- Now move forex history to the new directory and remove all old cache files
data_directory = db.user_data_dir.parent
price_history_dir = get_or_make_price_history_dir(data_directory)
forex_history_file = data_directory / 'price_history_forex.json'
if forex_history_file.is_file():
shutil.move(
forex_history_file, # type: ignore
price_history_dir / 'forex_history_file.json',
)
prefix = os.path.join(str(data_directory), 'price_history_')
prefix = prefix.replace('\\', '\\\\')
files_list = glob.glob(prefix + '*.json')
for file_ in files_list:
file_ = file_.replace('\\\\', '\\')
try:
Path(file_).unlink()
except OSError:
pass
def test_get_cached_data_metadata(data_dir, database):
"""Test that the get_cached_data_metadata function works correctly
and returns just the metadata by reading part ofthe file
"""
contents = """{"start_time": 1301536800, "end_time": 1301540400,
"data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298,
"volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298,
"low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}"""
price_history_dir = get_or_make_price_history_dir(data_dir)
with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f:
f.write(contents)
cc = Cryptocompare(data_directory=data_dir, database=database)
result = cc.get_cached_data_metadata(
from_asset=A_BTC,
to_asset=A_USD,
)
assert result is not None
assert result[0] == 1301536800
assert result[1] == 1301540400
def _get_cached_price(self, from_asset: Asset, to_asset: Asset,
date: str) -> Optional[Price]:
price_history_dir = get_or_make_price_history_dir(self.data_directory)
filename = (
price_history_dir /
f'{PRICE_HISTORY_FILE_PREFIX }{from_asset.identifier}_{to_asset.identifier}.json'
)
if not filename.is_file():
return None
with open(filename, 'r') as f:
try:
data: Dict[str, Price] = rlk_jsonloads_dict(f.read())
except JSONDecodeError:
return None
if not isinstance(data, dict):
return None
return data.get(date, None)
def test_get_cached_data_metadata(data_dir, database):
"""Test that the get_cached_data_metadata function works correctly
and returns just the metadata by reading part ofthe file
The json cache data in production are saved as one line files.
So here we also keep it in one line on purpose. The previous
regex we used failed with 1 line json file
"""
contents = """{"start_time": 1301536800, "end_time": 1301540400, "data": [{"time": 1301536800, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}, {"time": 1301540400, "close": 0.298, "high": 0.298, "low": 0.298, "open": 0.298, "volumefrom": 0.298, "volumeto": 0.298}]}""" # noqa: E501
price_history_dir = get_or_make_price_history_dir(data_dir)
with open(price_history_dir / f'{PRICE_HISTORY_FILE_PREFIX}BTC_USD.json', 'w') as f:
f.write(contents)
cc = Cryptocompare(data_directory=data_dir, database=database)
# make sure that _read_cachefile_metadata runs and they are read from file and not from memory
cc.price_history = {}
result = cc.get_cached_data_metadata(
from_asset=A_BTC,
to_asset=A_USD,
)
assert result is not None
assert result[0] == 1301536800
assert result[1] == 1301540400
def get_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
only_check_cache: bool,
) -> Optional[List[PriceHistoryEntry]]:
"""
Get historical hour price data from cryptocompare
Returns a sorted list of price entries.
If only_check_cache is True then if the data is not cached locally this will return None
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
cache_key = PairCacheKey(from_asset.identifier + '_' +
to_asset.identifier)
cached_data = self._got_cached_data_at_timestamp(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if cached_data is not None:
return cached_data.data
if only_check_cache:
return None
now_ts = ts_now()
# save time at start of the query, in case the query does not complete due to rate ilmit
self.last_histohour_query_ts = now_ts
if cache_key in self.price_history:
old_data = self.price_history[cache_key].data
if timestamp > self.price_history[cache_key].end_time:
# We have a cache but the requested timestamp does not hit it
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=self.price_history[cache_key].end_time,
)
if old_data[-1].time == new_data[0]['time']:
old_data = old_data[:-1]
new_history = deque([x._asdict() for x in old_data]) + new_data
else:
# only other possibility, timestamp < cached start_time
# Get all available data, even before to_timestamp
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=self.price_history[cache_key].start_time,
to_timestamp=Timestamp(0),
)
if new_data[-1]['time'] == old_data[0].time:
new_data.pop()
new_history = new_data + deque([x._asdict() for x in old_data])
calculated_history = list(new_history)
else:
calculated_history = list(
self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=Timestamp(0),
))
if len(calculated_history) == 0:
return [] # empty list means we found nothing
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# and now since we actually queried the data let's also cache them
filename = (get_or_make_price_history_dir(self.data_directory) /
(PRICE_HISTORY_FILE_PREFIX + cache_key + '.json'))
log.info(
'Updating price history cache',
filename=filename,
from_asset=from_asset,
to_asset=to_asset,
)
write_history_data_in_file(
data=calculated_history,
filepath=filename,
start_ts=calculated_history[0]['time'],
end_ts=now_ts,
)
# Finally save the objects in memory and return them
data_including_time = {
'data': calculated_history,
'start_time': calculated_history[0]['time'],
'end_time': now_ts,
}
self.price_history_file[cache_key] = filename
self.price_history[cache_key] = _dict_history_to_data(
data_including_time)
self.last_histohour_query_ts = ts_now(
) # also save when last query finished
return self.price_history[cache_key].data
