def _get_user_reserves(
self, address: ChecksumEthAddress) -> List[AaveUserReserve]:
query = self.graph.query(
querystr=USER_RESERVES_QUERY.format(address=address.lower()), )
query_v2 = self.graph_v2.query(
querystr=USER_RESERVES_QUERY.format(address=address.lower()), )
result = []
for entry in query['userReserves'] + query_v2['userReserves']:
reserve = entry['reserve']
try:
result.append(
AaveUserReserve(
# The ID of reserve is the address of the asset and the address of the market's LendingPoolAddressProvider, in lower case # noqa: E501
address=deserialize_ethereum_address(
reserve['id'][:42]),
symbol=reserve['symbol'],
))
except DeserializationError:
log.error(
f'Failed to deserialize reserve address {reserve["id"]} '
f'Skipping reserve address {reserve["id"]} for user address {address}',
)
continue
return result
def get_all_eth_token_info() -> List[EthTokenInfo]:
if AssetResolver().eth_token_info is not None:
return AssetResolver().eth_token_info # type: ignore
all_tokens = []
for identifier, asset_data in AssetResolver().assets.items():
# If an unknown asset is found (can happen if list is updated but code is not)
# then default to the "own chain" type"
asset_type = asset_type_mapping.get(asset_data['type'],
AssetType.OWN_CHAIN)
if asset_type not in (AssetType.ETH_TOKEN_AND_MORE,
AssetType.ETH_TOKEN):
continue
all_tokens.append(
EthTokenInfo(
identifier=identifier,
address=ChecksumEthAddress(asset_data['ethereum_address']),
symbol=asset_data['symbol'],
name=asset_data['name'],
decimals=int(asset_data['ethereum_token_decimals']),
))
AssetResolver().eth_token_info = all_tokens
return all_tokens
def hex_or_bytes_to_address(value: Union[bytes, str]) -> ChecksumEthAddress:
"""Turns a 32bit bytes/HexBytes or a hexstring into an address
May raise:
- ConversionError if it can't convert a value to an int or if an unexpected
type is given.
"""
hexstr = hex_or_bytes_to_str(value)
return ChecksumEthAddress(to_checksum_address('0x' + hexstr[26:]))
def deserialize_ethereum_address(symbol: str) -> ChecksumEthAddress:
"""This is identical to string_to_ethereum_address()
TODO:
But it's wrong. We should differentiate between those two functions.
That one should only be used for typing purposes while this one here
should be used to properly deserialize and check that symbol is indeed
an ethereum address and is always checksummed. So also external input sanitization.
https://github.com/rotki/rotki/issues/2334
"""
return ChecksumEthAddress(HexAddress(HexStr(symbol)))
def hex_or_bytes_to_address(value: Union[bytes, str]) -> ChecksumEthAddress:
"""Turns a 32bit bytes/HexBytes or a hexstring into an address
May raise:
- DeserializationError if it can't convert a value to an int or if an unexpected
type is given.
"""
try:
hexstr = hex_or_bytes_to_str(value)
except ConversionError as e:
raise DeserializationError(
f'Could not turn {value!r} to an ethereum address') from e
return ChecksumEthAddress(to_checksum_address('0x' + hexstr[26:]))
def _get_user_reserves(self, address: ChecksumEthAddress) -> List[AaveUserReserve]:
query = self.graph.query(
querystr=USER_RESERVES_QUERY.format(address=address.lower()),
)
result = []
for entry in query['userReserves']:
reserve = entry['reserve']
result.append(AaveUserReserve(
address=to_checksum_address(reserve['id']),
symbol=reserve['symbol'],
))
return result
def get_all_eth_tokens() -> List[EthTokenInfo]:
all_tokens = list()
for _, asset_data in AssetResolver().assets.items():
asset_type = asset_type_mapping[asset_data['type']]
if asset_type not in (AssetType.ETH_TOKEN_AND_MORE, AssetType.ETH_TOKEN):
continue
all_tokens.append(EthTokenInfo(
address=ChecksumEthAddress(asset_data['ethereum_address']),
symbol=asset_data['symbol'],
name=asset_data['name'],
decimal=int(asset_data['ethereum_token_decimals']),
))
return all_tokens
def get_common_params(
from_ts: Timestamp,
to_ts: Timestamp,
address: ChecksumEthAddress,
address_type: Literal['Bytes!', 'String!'] = 'Bytes!',
) -> Tuple[Dict[str, Any], Dict[str, Any]]:
param_types = {
'$start_ts': 'Int!',
'$end_ts': 'Int!',
'$address': address_type,
}
param_values = {
'start_ts': from_ts,
'end_ts': to_ts,
'address': address.lower(),
}
return param_types, param_values
def get_all_eth_token_info() -> List[EthTokenInfo]:
if AssetResolver().eth_token_info is not None:
return AssetResolver().eth_token_info # type: ignore
all_tokens = []
for identifier, asset_data in AssetResolver().assets.items():
asset_type = asset_type_mapping[asset_data['type']]
if asset_type not in (AssetType.ETH_TOKEN_AND_MORE, AssetType.ETH_TOKEN):
continue
all_tokens.append(EthTokenInfo(
identifier=identifier,
address=ChecksumEthAddress(asset_data['ethereum_address']),
symbol=asset_data['symbol'],
name=asset_data['name'],
decimals=int(asset_data['ethereum_token_decimals']),
))
AssetResolver().eth_token_info = all_tokens
return all_tokens
def test_update_conflicts(rotkehlchen_api_server, globaldb):
"""Test that conflicts in an asset update are handled properly"""
async_query = random.choice([False, True])
rotki = rotkehlchen_api_server.rest_api.rotkehlchen
update_1 = """INSERT INTO assets(identifier,type,name,symbol,started, swapped_for, coingecko, cryptocompare, details_reference) VALUES("121-ada-FADS-as", "F","A name","SYMBOL",NULL, NULL,"", "", "121-ada-FADS-as");INSERT INTO common_asset_details(asset_id, forked) VALUES("121-ada-FADS-as", "BTC");
*
INSERT INTO ethereum_tokens(address, decimals, protocol) VALUES("0x6B175474E89094C44Da98b954EedeAC495271d0F", 8, "maker");INSERT INTO assets(identifier,type, name, symbol,started, swapped_for, coingecko, cryptocompare, details_reference) VALUES("_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F", "C", "New Multi Collateral DAI", "NDAI", 1573672677, NULL, "dai", NULL, "0x6B175474E89094C44Da98b954EedeAC495271d0F");
*
INSERT INTO assets(identifier,type,name,symbol,started, swapped_for, coingecko, cryptocompare, details_reference) VALUES("DASH", "B","Dash","DASH",1337, NULL, "dash-coingecko", NULL, "DASH");INSERT INTO common_asset_details(asset_id, forked) VALUES("DASH", "BTC");
*
INSERT INTO ethereum_tokens(address, decimals, protocol) VALUES("0x1B175474E89094C44Da98b954EedeAC495271d0F", 18, NULL); INSERT INTO assets(identifier,type, name, symbol,started, swapped_for, coingecko, cryptocompare, details_reference) VALUES("_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F", "C", "Conflicting token", "CTK", 1573672677, NULL, "ctk", NULL, "0x1B175474E89094C44Da98b954EedeAC495271d0F");
*
""" # noqa: E501
globaldb.add_asset( # add a conflicting token
asset_id='_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F',
asset_type=AssetType.ETHEREUM_TOKEN,
data=EthereumToken.initialize(
address=ChecksumEthAddress('0x1B175474E89094C44Da98b954EedeAC495271d0F'),
decimals=12,
name='Conflicting token',
symbol='CTK',
started=None,
swapped_for=None,
coingecko='ctk',
cryptocompare=None,
protocol=None,
underlying_tokens=None,
),
)
globaldb.add_user_owned_assets([Asset('_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F')])
update_patch = mock_asset_updates(
original_requests_get=requests.get,
latest=999999991,
updates={"999999991": {
"changes": 3,
"min_schema_version": GLOBAL_DB_VERSION,
"max_schema_version": GLOBAL_DB_VERSION,
}},
sql_actions={"999999991": update_1},
)
globaldb.add_setting_value(ASSETS_VERSION_KEY, 999999990)
start_assets_num = len(globaldb.get_all_asset_data(mapping=False))
with update_patch:
response = requests.get(
api_url_for(
rotkehlchen_api_server,
'assetupdatesresource',
),
json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
)
result = outcome['result']
assert outcome['message'] == ''
else:
result = assert_proper_response_with_result(response)
assert result['local'] == 999999990
assert result['remote'] == 999999991
assert result['new_changes'] == 3
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'assetupdatesresource',
),
json={'async_query': async_query},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
)
assert outcome['message'] == 'Found conflicts during assets upgrade'
result = outcome['result']
else:
result = assert_proper_response_with_result(
response,
message='Found conflicts during assets upgrade',
status_code=HTTPStatus.CONFLICT,
)
# Make sure that nothing was committed
assert globaldb.get_setting_value(ASSETS_VERSION_KEY, None) == 999999990
assert len(globaldb.get_all_asset_data(mapping=False)) == start_assets_num
with pytest.raises(UnknownAsset):
Asset('121-ada-FADS-as')
errors = rotki.msg_aggregator.consume_errors()
warnings = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0, f'Found errors: {errors}'
assert len(warnings) == 0, f'Found warnings: {warnings}'
# See that we get 3 conflicts
expected_result = [{
'identifier': '_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F',
'local': {
'name': 'Multi Collateral Dai',
'symbol': 'DAI',
'asset_type': 'ethereum token',
'started': 1573672677,
'forked': None,
'swapped_for': None,
'ethereum_address': '0x6B175474E89094C44Da98b954EedeAC495271d0F',
'decimals': 18,
'cryptocompare': None,
'coingecko': 'dai',
'protocol': None,
},
'remote': {
'name': 'New Multi Collateral DAI',
'symbol': 'NDAI',
'asset_type': 'ethereum token',
'started': 1573672677,
'forked': None,
'swapped_for': None,
'ethereum_address': '0x6B175474E89094C44Da98b954EedeAC495271d0F',
'decimals': 8,
'cryptocompare': None,
'coingecko': 'dai',
'protocol': 'maker',
},
}, {
'identifier': 'DASH',
'local': {
'name': 'Dash',
'symbol': 'DASH',
'asset_type': 'own chain',
'started': 1390095618,
'forked': None,
'swapped_for': None,
'ethereum_address': None,
'decimals': None,
'cryptocompare': None,
'coingecko': 'dash',
'protocol': None,
},
'remote': {
'name': 'Dash',
'symbol': 'DASH',
'asset_type': 'own chain',
'started': 1337,
'forked': 'BTC',
'swapped_for': None,
'ethereum_address': None,
'decimals': None,
'cryptocompare': None,
'coingecko': 'dash-coingecko',
'protocol': None,
},
}, {
'identifier': '_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F',
'local': {
'asset_type': 'ethereum token',
'coingecko': 'ctk',
'cryptocompare': None,
'decimals': 12,
'ethereum_address': '0x1B175474E89094C44Da98b954EedeAC495271d0F',
'forked': None,
'name': 'Conflicting token',
'protocol': None,
'started': None,
'swapped_for': None,
'symbol': 'CTK',
},
'remote': {
'asset_type': 'ethereum token',
'coingecko': 'ctk',
'cryptocompare': None,
'decimals': 18,
'ethereum_address': '0x1b175474E89094C44DA98B954EeDEAC495271d0f',
'forked': None,
'name': 'Conflicting token',
'protocol': None,
'started': 1573672677,
'swapped_for': None,
'symbol': 'CTK',
},
}]
assert result == expected_result
# now try the update again but specify the conflicts resolution
conflicts = {'_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F': 'remote', 'DASH': 'local', '_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F': 'remote'} # noqa: E501
response = requests.post(
api_url_for(
rotkehlchen_api_server,
'assetupdatesresource',
),
json={'async_query': async_query, 'conflicts': conflicts},
)
if async_query:
task_id = assert_ok_async_response(response)
outcome = wait_for_async_task(
rotkehlchen_api_server,
task_id,
)
assert outcome['message'] == ''
result = outcome['result']
else:
result = assert_proper_response_with_result(
response,
message='',
status_code=HTTPStatus.OK,
)
cursor = globaldb._conn.cursor()
# check conflicts were solved as per the given choices and new asset also added
assert result is True
assert globaldb.get_setting_value(ASSETS_VERSION_KEY, None) == 999999991
errors = rotki.msg_aggregator.consume_errors()
warnings = rotki.msg_aggregator.consume_warnings()
assert len(errors) == 0, f'Found errors: {errors}'
assert len(warnings) == 0, f'Found warnings: {warnings}'
dai = EthereumToken('0x6B175474E89094C44Da98b954EedeAC495271d0F')
assert dai.identifier == strethaddress_to_identifier('0x6B175474E89094C44Da98b954EedeAC495271d0F') # noqa: E501
assert dai.name == 'New Multi Collateral DAI'
assert dai.symbol == 'NDAI'
assert dai.asset_type == AssetType.ETHEREUM_TOKEN
assert dai.started == 1573672677
assert dai.forked is None
assert dai.swapped_for is None
assert dai.coingecko == 'dai'
assert dai.cryptocompare is None
assert dai.ethereum_address == '0x6B175474E89094C44Da98b954EedeAC495271d0F'
assert dai.decimals == 8
assert dai.protocol == 'maker'
# make sure data is in both tables
assert cursor.execute('SELECT COUNT(*) from ethereum_tokens WHERE address="0x6B175474E89094C44Da98b954EedeAC495271d0F";').fetchone()[0] == 1 # noqa: E501
assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="_ceth_0x6B175474E89094C44Da98b954EedeAC495271d0F";').fetchone()[0] == 1 # noqa: E501
dash = Asset('DASH')
assert dash.identifier == 'DASH'
assert dash.name == 'Dash'
assert dash.symbol == 'DASH'
assert dash.asset_type == AssetType.OWN_CHAIN
assert dash.started == 1390095618
assert dash.forked is None
assert dash.swapped_for is None
assert dash.coingecko == 'dash'
assert dash.cryptocompare is None
assert cursor.execute('SELECT COUNT(*) from common_asset_details WHERE asset_id="DASH";').fetchone()[0] == 1 # noqa: E501
assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="DASH";').fetchone()[0] == 1 # noqa: E501
new_asset = Asset('121-ada-FADS-as')
assert new_asset.identifier == '121-ada-FADS-as'
assert new_asset.name == 'A name'
assert new_asset.symbol == 'SYMBOL'
assert new_asset.asset_type == AssetType.COUNTERPARTY_TOKEN
assert new_asset.started is None
assert new_asset.forked == 'BTC'
assert new_asset.swapped_for is None
assert new_asset.coingecko == ''
assert new_asset.cryptocompare == ''
assert cursor.execute('SELECT COUNT(*) from common_asset_details WHERE asset_id="121-ada-FADS-as";').fetchone()[0] == 1 # noqa: E501
assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="121-ada-FADS-as";').fetchone()[0] == 1 # noqa: E501
ctk = EthereumToken('0x1B175474E89094C44Da98b954EedeAC495271d0F')
assert ctk.name == 'Conflicting token'
assert ctk.symbol == 'CTK'
assert ctk.asset_type == AssetType.ETHEREUM_TOKEN
assert ctk.started == 1573672677
assert ctk.forked is None
assert ctk.swapped_for is None
assert ctk.coingecko == 'ctk'
assert ctk.cryptocompare is None
assert ctk.ethereum_address == '0x1B175474E89094C44Da98b954EedeAC495271d0F'
assert ctk.decimals == 18
assert ctk.protocol is None
assert cursor.execute('SELECT COUNT(*) from ethereum_tokens WHERE address="0x1B175474E89094C44Da98b954EedeAC495271d0F";').fetchone()[0] == 1 # noqa: E501
assert cursor.execute('SELECT COUNT(*) from assets WHERE identifier="_ceth_0x1B175474E89094C44Da98b954EedeAC495271d0F";').fetchone()[0] == 1 # noqa: E501
def _read_subgraph_trades(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the AMM subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to AMM lingo:
Trade type BUY:
- `asset1In` (QUOTE, reserve1) is gt 0.
- `asset0Out` (BASE, reserve0) is gt 0.
Trade type SELL:
- `asset0In` (BASE, reserve0) is gt 0.
- `asset1Out` (QUOTE, reserve1) is gt 0.
May raise
- RemoteError
"""
trades: List[AMMTrade] = []
query_id = '0'
query_offset = 0
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
'$id': 'ID!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
'id': query_id,
}
querystr = format_query_indentation(self.swaps_query.format())
while True:
try:
result = self.graph.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(
SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e),
location=self.location), )
raise
for entry in result['swaps']:
swaps = []
try:
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['pair']['token0']
swap_token1 = swap['pair']['token1']
try:
token0_deserialized = deserialize_ethereum_address(
swap_token0['id'])
token1_deserialized = deserialize_ethereum_address(
swap_token1['id'])
from_address_deserialized = deserialize_ethereum_address(
swap['sender']) # noqa
to_address_deserialized = deserialize_ethereum_address(
swap['to'])
except DeserializationError:
msg = (
f'Failed to deserialize addresses in trade from {self.location} graph' # noqa
f' with token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, ' # noqa
f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}'
)
log.error(msg)
continue
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token0['symbol'],
ethereum_address=token0_deserialized,
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token1['symbol'],
ethereum_address=token1_deserialized,
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
try:
amount0_in = FVal(swap['amount0In'])
amount1_in = FVal(swap['amount1In'])
amount0_out = FVal(swap['amount0Out'])
amount1_out = FVal(swap['amount1Out'])
except ValueError as e:
log.error(
f'Failed to read amounts in {self.location} swap {str(swap)}. '
f'{str(e)}.', )
continue
swaps.append(
AMMSwap(
tx_hash=swap['id'].split('-')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=from_address_deserialized,
to_address=to_address_deserialized,
timestamp=Timestamp(int(timestamp)),
location=self.location,
token0=token0,
token1=token1,
amount0_in=AssetAmount(amount0_in),
amount1_in=AssetAmount(amount1_in),
amount0_out=AssetAmount(amount0_out),
amount1_out=AssetAmount(amount1_out),
))
query_id = entry['id']
except KeyError as e:
log.error(
f'Failed to read trade in {self.location} swap {str(entry)}. '
f'{str(e)}.', )
continue
# with the new logic the list of swaps can be empty, in that case don't try
# to make trades from the swaps
if len(swaps) == 0:
continue
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result['swaps']) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
if query_offset == GRAPH_QUERY_SKIP_LIMIT:
query_offset = 0
new_query_id = query_id
else:
query_offset += GRAPH_QUERY_LIMIT
new_query_id = '0'
param_values = {
**param_values,
'id': new_query_id,
'offset': query_offset,
}
return trades
def test_associated_locations(database):
"""Test that locations imported in different places are correctly stored in database"""
# Add trades from different locations
trades = [Trade(
timestamp=Timestamp(1595833195),
location=Location.CRYPTOCOM,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1587825824),
location=Location.CRYPTOCOM,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596014214),
location=Location.BLOCKFI,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1565888464),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596014214),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1612051199),
location=Location.BLOCKFI,
base_asset=symbol_to_asset_or_token('USDC'),
quote_asset=symbol_to_asset_or_token('LTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('6404.6')),
rate=Price(FVal('151.6283999982779809352223797')),
fee=None,
fee_currency=None,
link='',
notes='One Time',
), Trade(
timestamp=Timestamp(1595833195),
location=Location.POLONIEX,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596429934),
location=Location.COINBASE,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00061475')),
rate=Price(FVal('309.0687271248474989833265555')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596429934),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
)]
# Add multiple entries for same exchange + connected exchange
database.add_trades(trades)
kraken_api_key1 = ApiKey('kraken_api_key')
kraken_api_secret1 = ApiSecret(b'kraken_api_secret')
kraken_api_key2 = ApiKey('kraken_api_key2')
kraken_api_secret2 = ApiSecret(b'kraken_api_secret2')
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
# add mock kraken and binance
database.add_exchange('kraken1', Location.KRAKEN, kraken_api_key1, kraken_api_secret1)
database.add_exchange('kraken2', Location.KRAKEN, kraken_api_key2, kraken_api_secret2)
database.add_exchange('binance', Location.BINANCE, binance_api_key, binance_api_secret)
# Add uniswap and sushiswap events
database.add_amm_events([
LiquidityPoolEvent(
tx_hash='0x47ea26957ce09e84a51b51dfdab6a4ac1c3672a372eef77b15ef7677174ac847',
log_index=23,
address=ChecksumEthAddress('0x3163Bb273E8D9960Ce003fD542bF26b4C529f515'),
timestamp=Timestamp(1590011534),
event_type=EventType.MINT_SUSHISWAP,
pool_address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
token0=EthereumToken('0x514910771AF9Ca656af840dff83E8264EcF986CA'),
token1=EthereumToken('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'),
amount0=FVal('3.313676003468974932'),
amount1=FVal('0.064189269269768657'),
usd_price=FVal('26.94433946158740371839009166230438'),
lp_amount=FVal('0.460858304063739927'),
),
])
database.add_amm_swaps([
AMMSwap(
tx_hash='0xa54bf4c68d435e3c8f432fd7e62b7f8aca497a831a3d3fca305a954484ddd7b2',
log_index=208,
address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
from_address=string_to_ethereum_address('0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F'),
to_address=string_to_ethereum_address('0xC9cB53B48A2f3A9e75982685644c1870F1405CCb'),
timestamp=Timestamp(1609301469),
location=Location.UNISWAP,
token0=EthereumToken('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'),
token1=EthereumToken('0xdAC17F958D2ee523a2206206994597C13D831ec7'),
amount0_in=AssetAmount(FVal('2.6455727132446468')),
amount1_in=AssetAmount(ZERO),
amount0_out=AssetAmount(ZERO),
amount1_out=AssetAmount(FVal('1936.810111')),
),
])
database.add_balancer_events([
BalancerEvent(
tx_hash='0xa54bf4c68d435e3c8f432fd7e62b7f8aca497a831a3d3fca305a954484ddd7b3',
log_index=23,
address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
timestamp=Timestamp(1609301469),
event_type=BalancerBPTEventType.MINT,
pool_address_token=EthereumToken('0x514910771AF9Ca656af840dff83E8264EcF986CA'),
lp_balance=Balance(amount=FVal(2), usd_value=FVal(3)),
amounts=[
AssetAmount(FVal(1)),
AssetAmount(FVal(2)),
],
),
])
expected_locations = {
Location.KRAKEN,
Location.BINANCE,
Location.BLOCKFI,
Location.NEXO,
Location.CRYPTOCOM,
Location.POLONIEX,
Location.COINBASE,
Location.EXTERNAL,
Location.SUSHISWAP,
Location.UNISWAP,
Location.BALANCER,
}
assert set(database.get_associated_locations()) == expected_locations
def _get_user_data(
self,
from_ts: Timestamp,
to_ts: Timestamp,
address: ChecksumEthAddress,
balances: AaveBalances,
) -> AaveHistory:
last_query = self.database.get_used_query_range(
f'aave_events_{address}')
db_events = self.database.get_aave_events(address=address)
now = ts_now()
last_query_ts = 0
if last_query is not None:
last_query_ts = last_query[1]
from_ts = Timestamp(last_query_ts + 1)
deposits = withdrawals = borrows = repays = liquidation_calls = []
query = self.graph.query(
querystr=USER_EVENTS_QUERY,
param_types={'$address': 'ID!'},
param_values={'address': address.lower()},
)
user_result = query['users'][0]
if now - last_query_ts > AAVE_GRAPH_RECENT_SECS:
# In theory if these were individual queries we should do them only if
# we have not queried recently. In practise since we only do 1 query above
# this is useless for now, but keeping the mechanism in case we change
# the way we query the subgraph
deposits = self._parse_deposits(user_result['depositHistory'],
from_ts, to_ts)
withdrawals = self._parse_withdrawals(
withdrawals=user_result['redeemUnderlyingHistory'],
from_ts=from_ts,
to_ts=to_ts,
)
borrows = self._parse_borrows(user_result['borrowHistory'],
from_ts, to_ts)
repays = self._parse_repays(user_result['repayHistory'], from_ts,
to_ts)
liquidation_calls = self._parse_liquidations(
user_result['liquidationCallHistory'],
from_ts,
to_ts,
)
result = self._process_events(
user_address=address,
user_result=user_result,
from_ts=from_ts,
to_ts=to_ts,
deposits=deposits,
withdrawals=withdrawals,
borrows=borrows,
repays=repays,
liquidations=liquidation_calls,
db_events=db_events,
balances=balances,
)
# Add all new events to the DB
new_events: List[
AaveEvent] = deposits + withdrawals + result.interest_events + borrows + repays + liquidation_calls # type: ignore # noqa: E501
self.database.add_aave_events(address, new_events)
# After all events have been queried then also update the query range.
# Even if no events are found for an address we need to remember the range
self.database.update_used_query_range(
name=f'aave_events_{address}',
start_ts=Timestamp(0),
end_ts=now,
)
# Sort actions so that actions with same time are sorted deposit -> interest -> withdrawal
all_events: List[AaveEvent] = new_events + db_events
sort_map = {
'deposit': 0,
'interest': 0.1,
'withdrawal': 0.2,
'borrow': 0.3,
'repay': 0.4,
'liquidation': 0.5
} # noqa: E501
all_events.sort(
key=lambda event: sort_map[event.event_type] + event.timestamp)
return AaveHistory(
events=all_events,
total_earned_interest=result.total_earned_interest,
total_lost=result.total_lost,
total_earned_liquidations=result.total_earned_liquidations,
)
def deserialize_ethereum_address(symbol: str) -> ChecksumEthAddress:
return ChecksumEthAddress(HexAddress(HexStr(symbol)))
def _get_trades_graph_for_address(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the Uniswap subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to Uniswap lingo:
Trade type BUY:
- `asset1In` (QUOTE, reserve1) is gt 0.
- `asset0Out` (BASE, reserve0) is gt 0.
Trade type SELL:
- `asset0In` (BASE, reserve0) is gt 0.
- `asset1Out` (QUOTE, reserve1) is gt 0.
"""
trades: List[AMMTrade] = []
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
}
querystr = format_query_indentation(SWAPS_QUERY.format())
while True:
result = self.graph.query( # type: ignore # caller already checks
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
result_data = result['swaps']
for entry in result_data:
swaps = []
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['pair']['token0']
swap_token1 = swap['pair']['token1']
token0 = get_ethereum_token(
symbol=swap_token0['symbol'],
ethereum_address=to_checksum_address(swap_token0['id']),
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_ethereum_token(
symbol=swap_token1['symbol'],
ethereum_address=to_checksum_address(swap_token1['id']),
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
amount0_in = FVal(swap['amount0In'])
amount1_in = FVal(swap['amount1In'])
amount0_out = FVal(swap['amount0Out'])
amount1_out = FVal(swap['amount1Out'])
swaps.append(AMMSwap(
tx_hash=swap['id'].split('-')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=to_checksum_address(swap['sender']),
to_address=to_checksum_address(swap['to']),
timestamp=Timestamp(int(timestamp)),
location=Location.UNISWAP,
token0=token0,
token1=token1,
amount0_in=AssetAmount(amount0_in),
amount1_in=AssetAmount(amount1_in),
amount0_out=AssetAmount(amount0_out),
amount1_out=AssetAmount(amount1_out),
))
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
param_values = {
**param_values,
'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore
}
return trades
def _get_events_graph(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
event_type: EventType,
) -> List[LiquidityPoolEvent]:
"""Get the address' events (mints & burns) querying the Uniswap subgraph
Each event data is stored in a <LiquidityPoolEvent>.
"""
address_events: List[LiquidityPoolEvent] = []
if event_type == EventType.MINT:
query = MINTS_QUERY
query_schema = 'mints'
elif event_type == EventType.BURN:
query = BURNS_QUERY
query_schema = 'burns'
else:
log.error(f'Unexpected event_type: {event_type}. Skipping events query.')
return address_events
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
}
querystr = format_query_indentation(query.format())
while True:
result = self.graph.query( # type: ignore # caller already checks
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
result_data = result[query_schema]
for event in result_data:
token0_ = event['pair']['token0']
token1_ = event['pair']['token1']
token0 = get_ethereum_token(
symbol=token0_['symbol'],
ethereum_address=to_checksum_address(token0_['id']),
name=token0_['name'],
decimals=token0_['decimals'],
)
token1 = get_ethereum_token(
symbol=token1_['symbol'],
ethereum_address=to_checksum_address(token1_['id']),
name=token1_['name'],
decimals=int(token1_['decimals']),
)
lp_event = LiquidityPoolEvent(
tx_hash=event['transaction']['id'],
log_index=int(event['logIndex']),
address=address,
timestamp=Timestamp(int(event['timestamp'])),
event_type=event_type,
pool_address=to_checksum_address(event['pair']['id']),
token0=token0,
token1=token1,
amount0=AssetAmount(FVal(event['amount0'])),
amount1=AssetAmount(FVal(event['amount1'])),
usd_price=Price(FVal(event['amountUSD'])),
lp_amount=AssetAmount(FVal(event['liquidity'])),
)
address_events.append(lp_event)
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
param_values = {
**param_values,
'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore
}
return address_events
def _get_events_graph(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
event_type: EventType,
) -> List[LiquidityPoolEvent]:
"""Get the address' events (mints & burns) querying the AMM's subgraph
Each event data is stored in a <LiquidityPoolEvent>.
"""
address_events: List[LiquidityPoolEvent] = []
if event_type == self.mint_event:
query = MINTS_QUERY
query_schema = 'mints'
elif event_type == self.burn_event:
query = BURNS_QUERY
query_schema = 'burns'
else:
log.error(
f'Unexpected {self.location} event_type: {event_type}. Skipping events query.',
)
return address_events
query_id = '0'
query_offset = 0
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
'$id': 'ID!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': query_offset,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
'id': query_id,
}
querystr = format_query_indentation(query.format())
while True:
try:
result = self.graph.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(
SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e),
location=self.location), )
raise
except AttributeError as e:
raise ModuleInitializationFailure(
f'{self.location} subgraph remote error') from e
result_data = result[query_schema]
for event in result_data:
token0_ = event['pair']['token0']
token1_ = event['pair']['token1']
try:
token0_deserialized = deserialize_ethereum_address(
token0_['id'])
token1_deserialized = deserialize_ethereum_address(
token1_['id'])
pool_deserialized = deserialize_ethereum_address(
event['pair']['id'])
except DeserializationError as e:
msg = (
f'Failed to deserialize address involved in liquidity pool event for'
f' {self.location}. Token 0: {token0_["id"]}, token 1: {token0_["id"]},'
f' pair: {event["pair"]["id"]}.')
log.error(msg)
raise RemoteError(msg) from e
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=token0_['symbol'],
ethereum_address=token0_deserialized,
name=token0_['name'],
decimals=token0_['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=token1_['symbol'],
ethereum_address=token1_deserialized,
name=token1_['name'],
decimals=int(token1_['decimals']),
)
lp_event = LiquidityPoolEvent(
tx_hash=event['transaction']['id'],
log_index=int(event['logIndex']),
address=address,
timestamp=Timestamp(int(event['timestamp'])),
event_type=event_type,
pool_address=pool_deserialized,
token0=token0,
token1=token1,
amount0=AssetAmount(FVal(event['amount0'])),
amount1=AssetAmount(FVal(event['amount1'])),
usd_price=Price(FVal(event['amountUSD'])),
lp_amount=AssetAmount(FVal(event['liquidity'])),
)
address_events.append(lp_event)
query_id = event['id']
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
if query_offset == GRAPH_QUERY_SKIP_LIMIT:
query_offset = 0
new_query_id = query_id
else:
query_offset += GRAPH_QUERY_LIMIT
new_query_id = '0'
param_values = {
**param_values,
'id': new_query_id,
'offset': query_offset,
}
return address_events
def address_to_bytes32(address: ChecksumEthAddress) -> str:
return '0x' + 24 * '0' + address.lower()[2:]
from dataclasses import asdict
from eth_typing import HexAddress, HexStr
from rotkehlchen.assets.asset import EthereumToken
from rotkehlchen.assets.unknown_asset import UnknownEthereumToken
from rotkehlchen.typing import ChecksumEthAddress
SHUF_ETHEREUM_ADDRESS = ChecksumEthAddress(
HexAddress(HexStr('0x3A9FfF453d50D4Ac52A6890647b823379ba36B9E')), )
SHUF_SYMBOL = 'SHUF'
SHUF_NAME = 'Shuffle.Monster V3'
SHUF_DECIMALS = 18
# Test initialization
def test_init_default():
ue_token = UnknownEthereumToken(
ethereum_address=SHUF_ETHEREUM_ADDRESS,
symbol=SHUF_SYMBOL,
)
assert ue_token.ethereum_address == SHUF_ETHEREUM_ADDRESS
assert ue_token.symbol == SHUF_SYMBOL
assert ue_token.name is None
assert ue_token.decimals is None
# Test operators
def test_eq():
ue_token_1 = UnknownEthereumToken(
ethereum_address=SHUF_ETHEREUM_ADDRESS,
def string_to_ethereum_address(value: str) -> ChecksumEthAddress:
"""This is a conversion without any checks of a string to ethereum address
Is only used for typing.
"""
return ChecksumEthAddress(HexAddress(HexStr(value)))
def _get_trades_graph_v3_for_address(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the Uniswap subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to Uniswap lingo:
Trade type BUY:
- `amount1` (QUOTE, reserve1) is gt 0.
- `amount0` (BASE, reserve0) is lt 0.
Trade type SELL:
- `amount0` (BASE, reserve0) is gt 0.
- `amount1` (QUOTE, reserve1) is lt 0.
May raise:
- RemoteError
"""
trades: List[AMMTrade] = []
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
}
querystr = format_query_indentation(V3_SWAPS_QUERY.format())
while True:
try:
result = self.graph_v3.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e)))
raise
result_data = result['swaps']
for entry in result_data:
swaps = []
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['token0']
swap_token1 = swap['token1']
try:
token0_deserialized = deserialize_ethereum_address(swap_token0['id'])
token1_deserialized = deserialize_ethereum_address(swap_token1['id'])
from_address_deserialized = deserialize_ethereum_address(swap['sender'])
to_address_deserialized = deserialize_ethereum_address(swap['recipient'])
except DeserializationError:
msg = (
f'Failed to deserialize addresses in trade from uniswap graph with '
f'token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, '
f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}'
)
log.error(msg)
continue
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token0['symbol'],
ethereum_address=token0_deserialized,
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token1['symbol'],
ethereum_address=token1_deserialized,
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
try:
if swap['amount0'].startswith('-'):
amount0_in = AssetAmount(FVal(ZERO))
amount0_out = deserialize_asset_amount_force_positive(swap['amount0'])
amount1_in = deserialize_asset_amount_force_positive(swap['amount1'])
amount1_out = AssetAmount(FVal(ZERO))
else:
amount0_in = deserialize_asset_amount_force_positive(swap['amount0'])
amount0_out = AssetAmount(FVal(ZERO))
amount1_in = AssetAmount(FVal(ZERO))
amount1_out = deserialize_asset_amount_force_positive(swap['amount1'])
except ValueError as e:
log.error(
f'Failed to read amounts in Uniswap V3 swap {str(swap)}. '
f'{str(e)}.',
)
continue
swaps.append(AMMSwap(
tx_hash=swap['id'].split('#')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=from_address_deserialized,
to_address=to_address_deserialized,
timestamp=Timestamp(int(timestamp)),
location=Location.UNISWAP,
token0=token0,
token1=token1,
amount0_in=amount0_in,
amount1_in=amount1_in,
amount0_out=amount0_out,
amount1_out=amount1_out,
))
# with the new logic the list of swaps can be empty, in that case don't try
# to make trades from the swaps
if len(swaps) == 0:
continue
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
param_values = {
**param_values,
'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore
}
return trades
