def transactions_from_dictlist(
given_transactions: List[Dict[str, Any]],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[EthereumTransaction]:
""" Gets a list of transaction, most probably read from the json files and
a time period. Returns it as a list of the transaction tuples that are inside the time period
May raise:
- KeyError: If the given_transactions contain data in an unexpected format
"""
returned_transactions = list()
for given_tx in given_transactions:
if given_tx['timestamp'] < start_ts:
continue
if given_tx['timestamp'] > end_ts:
break
timestamp = Timestamp(convert_to_int(given_tx['timestamp']))
tx_hash = given_tx['hash']
from_address = given_tx['from_address']
to_address = given_tx['to_address']
value = FVal(given_tx['value'])
gas = FVal(given_tx['gas'])
gas_price = FVal(given_tx['gas_price'])
gas_used = FVal(given_tx['gas_used'])
log.debug(
'Processing eth transaction',
sensitive_log=True,
timestamp=timestamp,
eth_tx_hash=tx_hash,
from_eth_address=from_address,
to_eth_address=to_address,
tx_value=value,
gas=gas,
gas_price=gas_price,
gas_used=gas_used,
)
returned_transactions.append(
EthereumTransaction(
timestamp=timestamp,
block_number=convert_to_int(given_tx['block_number']),
hash=tx_hash,
from_address=from_address,
to_address=to_address,
value=value,
gas=gas,
gas_price=gas_price,
gas_used=gas_used,
))
return returned_transactions
def deserialize_timestamp_from_kraken(time: Union[str, FVal]) -> Timestamp:
"""Deserializes a timestamp from a kraken api query result entry
Kraken has timestamps in floating point strings. Example: '1561161486.3056'.
If the dictionary has passed through rlk_jsonloads the entry can also be an Fval
Can throw DeserializationError if the data is not as expected
"""
if not time:
raise DeserializationError(
'Failed to deserialize a timestamp entry from a null entry in kraken',
)
if isinstance(time, str):
try:
return Timestamp(convert_to_int(time, accept_only_exact=False))
except ConversionError:
raise DeserializationError(
f'Failed to deserialize {time} kraken timestamp entry')
elif isinstance(time, FVal):
try:
return Timestamp(time.to_int(exact=False))
except ConversionError:
raise DeserializationError(
f'Failed to deserialize {time} kraken timestamp entry from an FVal',
)
else:
raise DeserializationError(
f'Failed to deserialize a timestamp entry from a {type(time)} entry in kraken',
)
def read_integer(data: Dict[str, Any], key: str) -> int:
try:
result = convert_to_int(data[key])
except ConversionError:
raise DeserializationError(
f'Failed to read {key} as an integer during etherscan transaction query',
)
return result
def read_integer(data: Dict[str, Any], key: str, api: str = DEFAULT_API) -> int:
try:
result = convert_to_int(data[key])
except ConversionError as e:
raise DeserializationError(
f'Failed to read {key} as an integer during {api} transaction query',
) from e
return result
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List:
with self.lock:
cache = self.check_trades_cache(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
if cache is not None:
result = cache
else:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = list()
for movement in result:
movements.append(
AssetMovement(
exchange='kraken',
category=movement['type'],
# Kraken timestamps have floating point
timestamp=convert_to_int(movement['time'],
accept_only_exact=False),
asset=asset_from_kraken(movement['asset']),
amount=FVal(movement['amount']),
fee=FVal(movement['fee']),
))
return movements
def _retrieve_price_from_data(
data: Optional[List[PriceHistoryEntry]],
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> Price:
"""Reads historical price data list returned from cryptocompare histohour
or cache and returns a price.
If nothing is found it returns Price(0). This can also happen if cryptocompare
returns a list of 0s for the timerange.
"""
price = Price(ZERO)
if data is None or len(data) == 0:
return price
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index_in_bounds = True
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
if index > len(data) - 1: # index out of bounds
# Try to see if index - 1 is there and if yes take it
if index > len(data):
index = index - 1
else: # give up. This happened: https://github.com/rotki/rotki/issues/1534
log.error(
f'Expected data index in cryptocompare historical hour price '
f'not found. Queried price of: {from_asset.identifier} in '
f'{to_asset.identifier} at {timestamp}. Data '
f'index: {index}. Length of returned data: {len(data)}. '
f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...',
)
index_in_bounds = False
if index_in_bounds:
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is not None and data[index].low is not None:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
return price
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
# Kraken timestamps have floating point
timestamp = Timestamp(
convert_to_int(kraken_trade['time'], accept_only_exact=False))
amount = FVal(kraken_trade['vol'])
cost = FVal(kraken_trade['cost'])
fee = FVal(kraken_trade['fee'])
order_type = trade_type_from_string(kraken_trade['type'])
rate = FVal(kraken_trade['price'])
if cost != amount * rate:
log.warning(
'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
location='kraken',
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
)
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
May raise:
- PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
try:
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
except UnsupportedAsset as e:
raise PriceQueryUnsupportedAsset(e.asset_name)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is None or data[index].low is None:
# If we get some None in the hourly set price to 0 so that we check alternatives
price = Price(ZERO)
else:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_list(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
result: List[Any]
if cache is not None:
result = cache
else:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(
self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
log.debug('Kraken deposit/withdrawals query result',
num_results=len(result))
movements = list()
for movement in result:
try:
movements.append(
AssetMovement(
exchange='kraken',
category=movement['type'],
# Kraken timestamps have floating point
timestamp=Timestamp(
convert_to_int(movement['time'],
accept_only_exact=False)),
asset=asset_from_kraken(movement['asset']),
amount=FVal(movement['amount']),
fee=Fee(FVal(movement['fee'])),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown kraken asset {e.asset_name}. '
f'Ignoring its deposit/withdrawals query.', )
continue
return movements
def test_convert_to_int():
assert convert_to_int('5') == 5
with pytest.raises(ConversionError):
assert convert_to_int(5.44, accept_only_exact=True) == 5
assert convert_to_int(5.44, accept_only_exact=False) == 5
assert convert_to_int(5.65, accept_only_exact=False) == 5
with pytest.raises(ConversionError):
assert convert_to_int(FVal('5.44'), accept_only_exact=True) == 5
assert convert_to_int(FVal('5.44'), accept_only_exact=False) == 5
assert convert_to_int(FVal('5.65'), accept_only_exact=False) == 5
assert convert_to_int(FVal('4'), accept_only_exact=True) == 4
assert convert_to_int(3) == 3
with pytest.raises(ConversionError):
assert convert_to_int('5.44', accept_only_exact=True) == 5
assert convert_to_int('5.44', accept_only_exact=False) == 5
assert convert_to_int('5.65', accept_only_exact=False) == 5
assert convert_to_int('4', accept_only_exact=False) == 4
with pytest.raises(ConversionError):
assert convert_to_int(b'5.44', accept_only_exact=True) == 5
assert convert_to_int(b'5.44', accept_only_exact=False) == 5
assert convert_to_int(b'5.65', accept_only_exact=False) == 5
assert convert_to_int(b'4', accept_only_exact=False) == 4
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
if from_asset in KNOWN_TO_MISS_FROM_CRYPTOCOMPARE:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is None or data[index].low is None:
# If we get some None in the hourly set price to 0 so that we check alternatives
price = Price(ZERO)
else:
price = (data[index].high + data[index].low) / 2
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = asset_btc_price * btc_to_asset_price
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset,
to_asset,
timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
May raise:
- PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
try:
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
except UnsupportedAsset as e:
raise PriceQueryUnsupportedAsset(e.asset_name)
price = Price(ZERO)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index_in_bounds = True
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
if index > len(data) - 1: # index out of bounds
# Try to see if index - 1 is there and if yes take it
if index > len(data):
index = index - 1
else: # give up. This happened: https://github.com/rotki/rotki/issues/1534
log.error(
f'Expected data index in cryptocompare historical hour price '
f'not found. Queried price of: {from_asset.identifier} in '
f'{to_asset.identifier} at {timestamp}. Data '
f'index: {index}. Length of returned data: {len(data)}. '
f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...',
)
index_in_bounds = False
if index_in_bounds:
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is not None and data[index].low is not None:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def test_convert_to_int():
assert convert_to_int('5') == 5
assert convert_to_int(
'37451082560000003241000000000003221111111111'
) == 37451082560000003241000000000003221111111111 # noqa: E501
with pytest.raises(ConversionError):
assert convert_to_int(5.44, accept_only_exact=True) == 5
assert convert_to_int(5.44, accept_only_exact=False) == 5
assert convert_to_int(5.65, accept_only_exact=False) == 5
with pytest.raises(ConversionError):
assert convert_to_int(FVal('5.44'), accept_only_exact=True) == 5
assert convert_to_int(FVal('5.44'), accept_only_exact=False) == 5
assert convert_to_int(FVal('5.65'), accept_only_exact=False) == 5
assert convert_to_int(FVal('4'), accept_only_exact=True) == 4
assert convert_to_int(3) == 3
with pytest.raises(ConversionError):
assert convert_to_int('5.44', accept_only_exact=True) == 5
assert convert_to_int('5.44', accept_only_exact=False) == 5
assert convert_to_int('5.65', accept_only_exact=False) == 5
assert convert_to_int('4', accept_only_exact=False) == 4
with pytest.raises(ConversionError):
assert convert_to_int(b'5.44', accept_only_exact=True) == 5
assert convert_to_int(b'5.44', accept_only_exact=False) == 5
assert convert_to_int(b'5.65', accept_only_exact=False) == 5
assert convert_to_int(b'4', accept_only_exact=False) == 4
def query_ethereum_txlist(
address: EthAddress,
internal: bool,
from_block: Optional[int] = None,
to_block: Optional[int] = None,
) -> List[EthereumTransaction]:
log.debug(
'Querying etherscan for tx list',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
)
result = list()
if internal:
reqstring = (
'https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address)
)
else:
reqstring = (
'https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address)
)
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = request_get_dict(reqstring)
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
log.error(
'Querying etherscan for tx list failed',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
error=resp['message'],
)
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp),
)
log.debug('Etherscan tx list query result', results_num=len(resp['result']))
for v in resp['result']:
# internal tx list contains no gasprice
gas_price = FVal(-1) if internal else FVal(v['gasPrice'])
result.append(EthereumTransaction(
timestamp=Timestamp(convert_to_int(v['timeStamp'])),
block_number=convert_to_int(v['blockNumber']),
hash=v['hash'],
from_address=v['from'],
to_address=v['to'],
value=FVal(v['value']),
gas=FVal(v['gas']),
gas_price=gas_price,
gas_used=FVal(v['gasUsed']),
))
return result
def query_ethereum_txlist(
address: EthAddress,
msg_aggregator: MessagesAggregator,
internal: bool,
from_block: Optional[int] = None,
to_block: Optional[int] = None,
) -> List[EthereumTransaction]:
"""Query ethereum tx list"""
log.debug(
'Querying etherscan for tx list',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
)
result = list()
if internal:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address))
else:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address))
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = request_get_dict(reqstring)
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
log.error(
'Querying etherscan for tx list failed',
sensitive_log=True,
internal=internal,
eth_address=address,
from_block=from_block,
to_block=to_block,
error=resp['message'],
)
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp), )
log.debug('Etherscan tx list query result',
results_num=len(resp['result']))
for v in resp['result']:
try:
tx = deserialize_transaction_from_etherscan(data=v,
internal=internal)
except DeserializationError as e:
msg_aggregator.add_warning(f'{str(e)}. Skipping transaction')
continue
result.append(tx)
return result
