def test_get_or_create_ethereum_token(globaldb, database):
cursor = globaldb._conn.cursor()
assets_num = cursor.execute('SELECT COUNT(*) from assets;').fetchone()[0]
assert A_DAI == get_or_create_ethereum_token(
userdb=database,
symbol='DAI',
ethereum_address='0x6B175474E89094C44Da98b954EedeAC495271d0F',
)
# Try getting a DAI token of a different address. Shold add new token to DB
new_token = get_or_create_ethereum_token(
userdb=database,
symbol='DAI',
ethereum_address='0xA379B8204A49A72FF9703e18eE61402FAfCCdD60',
)
assert cursor.execute(
'SELECT COUNT(*) from assets;').fetchone()[0] == assets_num + 1
assert new_token.symbol == 'DAI'
assert new_token.ethereum_address == '0xA379B8204A49A72FF9703e18eE61402FAfCCdD60'
# Try getting a symbol of normal chain with different address. Should add new token to DB
new_token = get_or_create_ethereum_token(
userdb=database,
symbol='DOT',
ethereum_address='0xB179B8204A49672FF9703e18eE61402FAfCCdD60',
)
assert new_token.symbol == 'DOT'
assert new_token.ethereum_address == '0xB179B8204A49672FF9703e18eE61402FAfCCdD60'
assert cursor.execute(
'SELECT COUNT(*) from assets;').fetchone()[0] == assets_num + 2
# Check that token with wrong symbol but existing address is returned
assert A_USDT == get_or_create_ethereum_token(
userdb=database,
symbol='ROFL',
ethereum_address='0xdAC17F958D2ee523a2206206994597C13D831ec7',
)
assert cursor.execute(
'SELECT COUNT(*) from assets;').fetchone()[0] == assets_num + 2
def _decode_result(
userdb: 'DBHandler',
data: Tuple,
known_assets: Set[EthereumToken],
unknown_assets: Set[EthereumToken],
) -> LiquidityPool:
pool_token = _decode_token(data[0])
token0 = _decode_token(data[1][0])
token1 = _decode_token(data[1][1])
assets = []
for token in (token0, token1):
asset = get_or_create_ethereum_token(
userdb=userdb,
symbol=token.symbol,
ethereum_address=token.address,
name=token.name,
decimals=token.decimals,
)
# Classify the asset either as price known or unknown
if asset.has_oracle():
known_assets.add(asset)
else:
unknown_assets.add(asset)
assets.append(
LiquidityPoolAsset(
asset=asset,
total_amount=None,
user_balance=Balance(amount=token.amount),
))
pool = LiquidityPool(
address=pool_token.address,
assets=assets,
total_supply=None,
user_balance=Balance(amount=pool_token.amount),
)
return pool
def _read_subgraph_trades(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the AMM subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to AMM lingo:
Trade type BUY:
- `asset1In` (QUOTE, reserve1) is gt 0.
- `asset0Out` (BASE, reserve0) is gt 0.
Trade type SELL:
- `asset0In` (BASE, reserve0) is gt 0.
- `asset1Out` (QUOTE, reserve1) is gt 0.
May raise
- RemoteError
"""
trades: List[AMMTrade] = []
query_id = '0'
query_offset = 0
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
'$id': 'ID!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
'id': query_id,
}
querystr = format_query_indentation(self.swaps_query.format())
while True:
try:
result = self.graph.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(
SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e),
location=self.location), )
raise
for entry in result['swaps']:
swaps = []
try:
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['pair']['token0']
swap_token1 = swap['pair']['token1']
try:
token0_deserialized = deserialize_ethereum_address(
swap_token0['id'])
token1_deserialized = deserialize_ethereum_address(
swap_token1['id'])
from_address_deserialized = deserialize_ethereum_address(
swap['sender']) # noqa
to_address_deserialized = deserialize_ethereum_address(
swap['to'])
except DeserializationError:
msg = (
f'Failed to deserialize addresses in trade from {self.location} graph' # noqa
f' with token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, ' # noqa
f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}'
)
log.error(msg)
continue
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token0['symbol'],
ethereum_address=token0_deserialized,
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token1['symbol'],
ethereum_address=token1_deserialized,
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
try:
amount0_in = FVal(swap['amount0In'])
amount1_in = FVal(swap['amount1In'])
amount0_out = FVal(swap['amount0Out'])
amount1_out = FVal(swap['amount1Out'])
except ValueError as e:
log.error(
f'Failed to read amounts in {self.location} swap {str(swap)}. '
f'{str(e)}.', )
continue
swaps.append(
AMMSwap(
tx_hash=swap['id'].split('-')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=from_address_deserialized,
to_address=to_address_deserialized,
timestamp=Timestamp(int(timestamp)),
location=self.location,
token0=token0,
token1=token1,
amount0_in=AssetAmount(amount0_in),
amount1_in=AssetAmount(amount1_in),
amount0_out=AssetAmount(amount0_out),
amount1_out=AssetAmount(amount1_out),
))
query_id = entry['id']
except KeyError as e:
log.error(
f'Failed to read trade in {self.location} swap {str(entry)}. '
f'{str(e)}.', )
continue
# with the new logic the list of swaps can be empty, in that case don't try
# to make trades from the swaps
if len(swaps) == 0:
continue
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result['swaps']) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
if query_offset == GRAPH_QUERY_SKIP_LIMIT:
query_offset = 0
new_query_id = query_id
else:
query_offset += GRAPH_QUERY_LIMIT
new_query_id = '0'
param_values = {
**param_values,
'id': new_query_id,
'offset': query_offset,
}
return trades
def _get_balances_graph(
self,
addresses: List[ChecksumEthAddress],
) -> ProtocolBalance:
"""Get the addresses' pools data querying this AMM's subgraph
Each liquidity position is converted into a <LiquidityPool>.
"""
address_balances: DDAddressToLPBalances = defaultdict(list)
known_assets: Set[EthereumToken] = set()
unknown_assets: Set[EthereumToken] = set()
addresses_lower = [address.lower() for address in addresses]
querystr = format_query_indentation(LIQUIDITY_POSITIONS_QUERY.format())
query_id = '0'
query_offset = 0
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$addresses': '[String!]',
'$balance': 'BigDecimal!',
'$id': 'ID!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'addresses': addresses_lower,
'balance': '0',
'id': query_id,
}
while True:
try:
result = self.graph.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(
SUBGRAPH_REMOTE_ERROR_MSG.format(
error_msg=str(e),
location=self.location,
), )
raise
result_data = result['liquidityPositions']
for lp in result_data:
lp_pair = lp['pair']
lp_total_supply = FVal(lp_pair['totalSupply'])
user_lp_balance = FVal(lp['liquidityTokenBalance'])
try:
user_address = deserialize_ethereum_address(
lp['user']['id'])
lp_address = deserialize_ethereum_address(lp_pair['id'])
except DeserializationError as e:
msg = (
f'Failed to Deserialize address. Skipping {self.location} '
f'pool {lp_pair} with user address {lp["user"]["id"]}')
log.error(msg)
raise RemoteError(msg) from e
# Insert LP tokens reserves within tokens dicts
token0 = lp_pair['token0']
token0['total_amount'] = lp_pair['reserve0']
token1 = lp_pair['token1']
token1['total_amount'] = lp_pair['reserve1']
liquidity_pool_assets = []
for token in token0, token1:
try:
deserialized_eth_address = deserialize_ethereum_address(
token['id'])
except DeserializationError as e:
msg = (
f'Failed to deserialize token address {token["id"]} '
f'Bad token address in {self.location} lp pair came from the graph.'
)
log.error(msg)
raise RemoteError(msg) from e
asset = get_or_create_ethereum_token(
userdb=self.database,
symbol=token['symbol'],
ethereum_address=deserialized_eth_address,
name=token['name'],
decimals=int(token['decimals']),
)
if asset.has_oracle():
known_assets.add(asset)
else:
unknown_assets.add(asset)
# Estimate the underlying asset total_amount
asset_total_amount = FVal(token['total_amount'])
user_asset_balance = (user_lp_balance / lp_total_supply *
asset_total_amount)
liquidity_pool_asset = LiquidityPoolAsset(
asset=asset,
total_amount=asset_total_amount,
user_balance=Balance(amount=user_asset_balance),
)
liquidity_pool_assets.append(liquidity_pool_asset)
liquidity_pool = LiquidityPool(
address=lp_address,
assets=liquidity_pool_assets,
total_supply=lp_total_supply,
user_balance=Balance(amount=user_lp_balance),
)
address_balances[user_address].append(liquidity_pool)
query_id = lp['id']
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
if query_offset == GRAPH_QUERY_SKIP_LIMIT:
query_offset = 0
new_query_id = query_id
else:
query_offset += GRAPH_QUERY_LIMIT
new_query_id = '0'
param_values = {
**param_values,
'id': new_query_id,
'offset': query_offset,
}
protocol_balance = ProtocolBalance(
address_balances=dict(address_balances),
known_assets=known_assets,
unknown_assets=unknown_assets,
)
return protocol_balance
def _maybe_decode_erc20_721_transfer(
self,
token: Optional[EthereumToken],
tx_log: EthereumTxReceiptLog,
transaction: EthereumTransaction,
decoded_events: List[HistoryBaseEntry], # pylint: disable=unused-argument
action_items: List[ActionItem],
) -> Optional[HistoryBaseEntry]:
if tx_log.topics[0] != ERC20_OR_ERC721_TRANSFER:
return None
if token is None:
try:
found_token = get_or_create_ethereum_token(
userdb=self.database,
ethereum_address=tx_log.address,
ethereum_manager=self.ethereum_manager,
)
except NotERC20Conformant:
return None # ignore non-ERC20 transfers for now
else:
found_token = token
transfer = self.base.decode_erc20_721_transfer(
token=found_token,
tx_log=tx_log,
transaction=transaction,
)
if transfer is None:
return None
for idx, action_item in enumerate(action_items):
if action_item.asset == found_token and action_item.amount == transfer.balance.amount and action_item.from_event_type == transfer.event_type and action_item.from_event_subtype == transfer.event_subtype: # noqa: E501
if action_item.action == 'skip':
action_items.pop(idx)
return None
# else atm only transform
if action_item.to_event_type is not None:
transfer.event_type = action_item.to_event_type
if action_item.to_event_subtype is not None:
transfer.event_subtype = action_item.to_event_subtype
if action_item.to_notes is not None:
transfer.notes = action_item.to_notes
if action_item.to_counterparty is not None:
transfer.counterparty = action_item.to_counterparty
if action_item.extra_data is not None:
transfer.extra_data = action_item.extra_data
if action_item.paired_event_data is not None:
# If there is a paired event to this, take care of the order
out_event = transfer
in_event = action_item.paired_event_data[0]
if action_item.paired_event_data[1] is True:
out_event = action_item.paired_event_data[0]
in_event = transfer
maybe_reshuffle_events(
out_event=out_event,
in_event=in_event,
events_list=decoded_events + [transfer],
)
action_items.pop(idx)
break # found an action item and acted on it
# Add additional information to transfers for different protocols
self._enrich_protocol_tranfers(
token=found_token,
tx_log=tx_log,
transaction=transaction,
event=transfer,
action_items=action_items,
)
return transfer
def _get_events_graph(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
event_type: EventType,
) -> List[LiquidityPoolEvent]:
"""Get the address' events (mints & burns) querying the AMM's subgraph
Each event data is stored in a <LiquidityPoolEvent>.
"""
address_events: List[LiquidityPoolEvent] = []
if event_type == self.mint_event:
query = MINTS_QUERY
query_schema = 'mints'
elif event_type == self.burn_event:
query = BURNS_QUERY
query_schema = 'burns'
else:
log.error(
f'Unexpected {self.location} event_type: {event_type}. Skipping events query.',
)
return address_events
query_id = '0'
query_offset = 0
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
'$id': 'ID!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': query_offset,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
'id': query_id,
}
querystr = format_query_indentation(query.format())
while True:
try:
result = self.graph.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(
SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e),
location=self.location), )
raise
except AttributeError as e:
raise ModuleInitializationFailure(
f'{self.location} subgraph remote error') from e
result_data = result[query_schema]
for event in result_data:
token0_ = event['pair']['token0']
token1_ = event['pair']['token1']
try:
token0_deserialized = deserialize_ethereum_address(
token0_['id'])
token1_deserialized = deserialize_ethereum_address(
token1_['id'])
pool_deserialized = deserialize_ethereum_address(
event['pair']['id'])
except DeserializationError as e:
msg = (
f'Failed to deserialize address involved in liquidity pool event for'
f' {self.location}. Token 0: {token0_["id"]}, token 1: {token0_["id"]},'
f' pair: {event["pair"]["id"]}.')
log.error(msg)
raise RemoteError(msg) from e
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=token0_['symbol'],
ethereum_address=token0_deserialized,
name=token0_['name'],
decimals=token0_['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=token1_['symbol'],
ethereum_address=token1_deserialized,
name=token1_['name'],
decimals=int(token1_['decimals']),
)
lp_event = LiquidityPoolEvent(
tx_hash=event['transaction']['id'],
log_index=int(event['logIndex']),
address=address,
timestamp=Timestamp(int(event['timestamp'])),
event_type=event_type,
pool_address=pool_deserialized,
token0=token0,
token1=token1,
amount0=AssetAmount(FVal(event['amount0'])),
amount1=AssetAmount(FVal(event['amount1'])),
usd_price=Price(FVal(event['amountUSD'])),
lp_amount=AssetAmount(FVal(event['liquidity'])),
)
address_events.append(lp_event)
query_id = event['id']
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
if query_offset == GRAPH_QUERY_SKIP_LIMIT:
query_offset = 0
new_query_id = query_id
else:
query_offset += GRAPH_QUERY_LIMIT
new_query_id = '0'
param_values = {
**param_values,
'id': new_query_id,
'offset': query_offset,
}
return address_events
def deserialize_bpt_event(
userdb: 'DBHandler',
raw_event: Dict[str, Any],
event_type: Literal[BalancerBPTEventType.MINT, BalancerBPTEventType.BURN],
) -> BalancerBPTEvent:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_event['id'])
raw_user_address = raw_event['user']['id']
amount = deserialize_asset_amount(raw_event['amount'])
raw_pool = raw_event['pool']
raw_pool_address = raw_pool['id']
raw_pool_tokens = raw_pool['tokens']
total_weight = deserialize_asset_amount(raw_pool['totalWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if total_weight == ZERO:
raise DeserializationError('Pool weight is zero.')
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
underlying_tokens = []
for raw_token in raw_pool_tokens:
try:
raw_token_address = raw_token['address']
token_symbol = raw_token['symbol']
token_name = raw_token['name']
token_decimals = raw_token['decimals']
token_weight = deserialize_asset_amount(raw_token['denormWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(raw_token_address)
token = get_or_create_ethereum_token(
userdb=userdb,
symbol=token_symbol,
ethereum_address=token_address,
name=token_name,
decimals=token_decimals,
)
underlying_tokens.append(UnderlyingToken(
address=token.ethereum_address,
weight=token_weight / total_weight,
))
underlying_tokens.sort(key=lambda x: x.address)
pool_address_token = get_or_create_ethereum_token(
userdb=userdb,
ethereum_address=pool_address,
symbol='BPT',
protocol='balancer',
decimals=18, # all BPT tokens have 18 decimals
underlying_tokens=underlying_tokens,
form_with_incomplete_data=True, # since some may not have decimals input correctly
)
bpt_event = BalancerBPTEvent(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
event_type=event_type,
pool_address_token=pool_address_token,
amount=amount,
)
return bpt_event
def deserialize_swap(userdb: 'DBHandler', raw_swap: Dict[str, Any]) -> AMMSwap:
"""May raise DeserializationError"""
try:
tx_hash, log_index = deserialize_transaction_id(raw_swap['id'])
timestamp = deserialize_timestamp(raw_swap['timestamp'])
raw_tokens = raw_swap['poolAddress']['tokens']
token0_symbol = raw_swap['tokenInSym']
token1_symbol = raw_swap['tokenOutSym']
amount0_in = deserialize_asset_amount(raw_swap['tokenAmountIn'])
amount1_out = deserialize_asset_amount(raw_swap['tokenAmountOut'])
raw_user_address = raw_swap['userAddress']['id'] # address
raw_caller_address = raw_swap['caller'] # from_address
raw_pool_address = raw_swap['poolAddress']['id'] # to_address
raw_token_in_address = raw_swap['tokenIn'] # token0_address
raw_token_out_address = raw_swap['tokenOut'] # token1_address
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if amount0_in == ZERO:
# Prevent a division by zero error when creating the trade
raise DeserializationError('TokenAmountIn balance is zero.')
# Checksum addresses
user_address = deserialize_ethereum_address(raw_user_address)
caller_address = deserialize_ethereum_address(raw_caller_address)
pool_address = deserialize_ethereum_address(raw_pool_address)
token_in_address = deserialize_ethereum_address(raw_token_in_address)
token_out_address = deserialize_ethereum_address(raw_token_out_address)
# Get token0 and token1
# When the controller removes all the tokens from a pool, `raw_tokens` will
# be an empty list. Therefore it won't be possible to get their names and
# decimals.
if len(raw_tokens) != 0:
try:
raw_address_tokens = {raw_token['address']: raw_token for raw_token in raw_tokens}
raw_token0 = raw_address_tokens.get(raw_token_in_address)
raw_token1 = raw_address_tokens.get(raw_token_out_address)
if raw_token0 is not None:
token0_name = raw_token0['name']
token0_decimals = raw_token0['decimals']
else:
token0_name = None
token0_decimals = None
if raw_token1 is not None:
token1_name = raw_token1['name']
token1_decimals = raw_token1['decimals']
else:
token1_name = None
token1_decimals = None
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
else:
token0_name = None
token0_decimals = None
token1_name = None
token1_decimals = None
token0 = get_or_create_ethereum_token(
userdb=userdb,
symbol=token0_symbol,
ethereum_address=token_in_address,
name=token0_name,
decimals=token0_decimals,
)
token1 = get_or_create_ethereum_token(
userdb=userdb,
symbol=token1_symbol,
ethereum_address=token_out_address,
name=token1_name,
decimals=token1_decimals,
)
amm_swap = AMMSwap(
tx_hash=tx_hash,
log_index=log_index,
address=user_address,
from_address=caller_address,
to_address=pool_address,
timestamp=timestamp,
location=Location.BALANCER,
token0=token0,
token1=token1,
amount0_in=amount0_in,
amount1_in=AssetAmount(ZERO),
amount0_out=AssetAmount(ZERO),
amount1_out=amount1_out,
)
return amm_swap
def deserialize_pool_share(
userdb: 'DBHandler',
raw_pool_share: Dict[str, Any],
) -> Tuple[ChecksumEthAddress, BalancerPoolBalance]:
"""May raise DeserializationError"""
try:
raw_user_address = raw_pool_share['userAddress']['id']
user_amount = deserialize_asset_amount(raw_pool_share['balance'])
raw_pool = raw_pool_share['poolId']
total_amount = deserialize_asset_amount(raw_pool['totalShares'])
raw_address = raw_pool['id']
raw_tokens = raw_pool['tokens']
total_weight = deserialize_asset_amount(raw_pool['totalWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
if total_weight == ZERO:
raise DeserializationError('Pool weight is zero.')
user_address = deserialize_ethereum_address(raw_user_address)
pool_address = deserialize_ethereum_address(raw_address)
pool_tokens = []
pool_token_balances = []
for raw_token in raw_tokens:
try:
raw_token_address = raw_token['address']
token_symbol = raw_token['symbol']
token_name = raw_token['name']
token_decimals = raw_token['decimals']
token_total_amount = deserialize_asset_amount(raw_token['balance'])
token_weight = deserialize_asset_amount(raw_token['denormWeight'])
except KeyError as e:
raise DeserializationError(f'Missing key: {str(e)}.') from e
token_address = deserialize_ethereum_address(raw_token_address)
token = get_or_create_ethereum_token(
userdb=userdb,
symbol=token_symbol,
ethereum_address=token_address,
name=token_name,
decimals=token_decimals,
)
if token_total_amount == ZERO:
raise DeserializationError(f'Token {token.identifier} balance is zero.')
weight = token_weight * 100 / total_weight
token_user_amount = user_amount / total_amount * token_total_amount
pool_token_balance = BalancerPoolTokenBalance(
token=token,
total_amount=token_total_amount,
user_balance=Balance(amount=token_user_amount),
weight=weight,
)
pool_token_balances.append(pool_token_balance)
pool_token = UnderlyingToken(address=token.ethereum_address, weight=weight / 100)
pool_tokens.append(pool_token)
pool_tokens.sort(key=lambda x: x.address)
pool_token_balances.sort(key=lambda x: x.token.ethereum_address)
balancer_pool_token = get_or_create_ethereum_token(
userdb=userdb,
symbol='BPT',
ethereum_address=pool_address,
protocol='balancer',
decimals=18, # All BPT tokens have 18 decimals
underlying_tokens=pool_tokens,
form_with_incomplete_data=True, # since some may not have had decimals input correctly
)
pool = BalancerPoolBalance(
pool_token=balancer_pool_token,
underlying_tokens_balance=pool_token_balances,
total_amount=total_amount,
user_balance=Balance(amount=user_amount),
)
return user_address, pool
def _get_trades_graph_v3_for_address(
self,
address: ChecksumEthAddress,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AMMTrade]:
"""Get the address' trades data querying the Uniswap subgraph
Each trade (swap) instantiates an <AMMTrade>.
The trade pair (i.e. BASE_QUOTE) is determined by `reserve0_reserve1`.
Translated to Uniswap lingo:
Trade type BUY:
- `amount1` (QUOTE, reserve1) is gt 0.
- `amount0` (BASE, reserve0) is lt 0.
Trade type SELL:
- `amount0` (BASE, reserve0) is gt 0.
- `amount1` (QUOTE, reserve1) is lt 0.
May raise:
- RemoteError
"""
trades: List[AMMTrade] = []
param_types = {
'$limit': 'Int!',
'$offset': 'Int!',
'$address': 'Bytes!',
'$start_ts': 'BigInt!',
'$end_ts': 'BigInt!',
}
param_values = {
'limit': GRAPH_QUERY_LIMIT,
'offset': 0,
'address': address.lower(),
'start_ts': str(start_ts),
'end_ts': str(end_ts),
}
querystr = format_query_indentation(V3_SWAPS_QUERY.format())
while True:
try:
result = self.graph_v3.query(
querystr=querystr,
param_types=param_types,
param_values=param_values,
)
except RemoteError as e:
self.msg_aggregator.add_error(SUBGRAPH_REMOTE_ERROR_MSG.format(error_msg=str(e)))
raise
result_data = result['swaps']
for entry in result_data:
swaps = []
for swap in entry['transaction']['swaps']:
timestamp = swap['timestamp']
swap_token0 = swap['token0']
swap_token1 = swap['token1']
try:
token0_deserialized = deserialize_ethereum_address(swap_token0['id'])
token1_deserialized = deserialize_ethereum_address(swap_token1['id'])
from_address_deserialized = deserialize_ethereum_address(swap['sender'])
to_address_deserialized = deserialize_ethereum_address(swap['recipient'])
except DeserializationError:
msg = (
f'Failed to deserialize addresses in trade from uniswap graph with '
f'token 0: {swap_token0["id"]}, token 1: {swap_token1["id"]}, '
f'swap sender: {swap["sender"]}, swap receiver {swap["to"]}'
)
log.error(msg)
continue
token0 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token0['symbol'],
ethereum_address=token0_deserialized,
name=swap_token0['name'],
decimals=swap_token0['decimals'],
)
token1 = get_or_create_ethereum_token(
userdb=self.database,
symbol=swap_token1['symbol'],
ethereum_address=token1_deserialized,
name=swap_token1['name'],
decimals=int(swap_token1['decimals']),
)
try:
if swap['amount0'].startswith('-'):
amount0_in = AssetAmount(FVal(ZERO))
amount0_out = deserialize_asset_amount_force_positive(swap['amount0'])
amount1_in = deserialize_asset_amount_force_positive(swap['amount1'])
amount1_out = AssetAmount(FVal(ZERO))
else:
amount0_in = deserialize_asset_amount_force_positive(swap['amount0'])
amount0_out = AssetAmount(FVal(ZERO))
amount1_in = AssetAmount(FVal(ZERO))
amount1_out = deserialize_asset_amount_force_positive(swap['amount1'])
except ValueError as e:
log.error(
f'Failed to read amounts in Uniswap V3 swap {str(swap)}. '
f'{str(e)}.',
)
continue
swaps.append(AMMSwap(
tx_hash=swap['id'].split('#')[0],
log_index=int(swap['logIndex']),
address=address,
from_address=from_address_deserialized,
to_address=to_address_deserialized,
timestamp=Timestamp(int(timestamp)),
location=Location.UNISWAP,
token0=token0,
token1=token1,
amount0_in=amount0_in,
amount1_in=amount1_in,
amount0_out=amount0_out,
amount1_out=amount1_out,
))
# with the new logic the list of swaps can be empty, in that case don't try
# to make trades from the swaps
if len(swaps) == 0:
continue
# Now that we got all swaps for a transaction, create the trade object
trades.extend(self._tx_swaps_to_trades(swaps))
# Check whether an extra request is needed
if len(result_data) < GRAPH_QUERY_LIMIT:
break
# Update pagination step
param_values = {
**param_values,
'offset': param_values['offset'] + GRAPH_QUERY_LIMIT, # type: ignore
}
return trades
def query_token_spam_list(db: 'DBHandler') -> Set[EthereumToken]:
"""Generate a set of assets that can be ignored combining information of cryptoscamdb
and the list of spam assets KNOWN_ETH_SPAM_TOKENS. This function also makes sure to get the
bad assets in the list of cryptoscamdb and ensures that they exists in the globaldb before
trying to add them.
TODO
This function tries to add as assets to the globaldb the tokens listed in
KNOWN_ETH_SPAM_TOKENS and not the ones coming from cryptoscamdb. The reason is that until the
v2 of the API the response contains both spam addresses and tokens and there is no way to know
if the address is for a contract or not. Checking if the address is a contract takes too much
time. When V2 gets released this can be fixed.
May raise:
- RemoteError
"""
try:
response = requests.get(
url='https://api.cryptoscamdb.org/v1/addresses',
timeout=DEFAULT_TIMEOUT_TUPLE,
)
data = response.json()
success, tokens_info = data['success'], data['result']
except requests.exceptions.RequestException as e:
raise RemoteError(
f'Failed to retrieve information from cryptoscamdb. {str(e)}'
) from e
except (DeserializationError, JSONDecodeError) as e:
raise RemoteError(
f'Failed to deserialize data from cryptoscamdb. {str(e)}') from e
except KeyError as e:
raise RemoteError(
f'Response from cryptoscamdb doesn\'t contain expected key. {str(e)}',
) from e
if success is False:
log.error(f'Failed to deserialize data from cryptoscamdb. {data}')
raise RemoteError(
'Failed to deserialize data from cryptoscamdb. Check the logs '
'to get more information', )
tokens_to_ignore = set()
for token_addr, token_data in tokens_info.items():
if not token_addr.startswith('0x') or token_data[0]['type'] != 'scam':
continue
try:
checksumed_address = to_checksum_address(token_addr)
except ValueError as e:
log.debug(f'Failed to read address from cryptoscamdb. {str(e)}')
continue
try:
token = EthereumToken(checksumed_address)
except UnknownAsset:
continue
if token is not None:
tokens_to_ignore.add(token)
# Try to add custom list
for token_address, info in KNOWN_ETH_SPAM_TOKENS.items():
try:
own_token = get_or_create_ethereum_token(
userdb=db,
ethereum_address=token_address,
protocol=SPAM_PROTOCOL,
form_with_incomplete_data=True,
decimals=info.get('decimals', 18),
name=info.get('name', MISSING_NAME_SPAM_TOKEN),
symbol=info.get('symbol', MISSING_SYMBOL_SPAM_TOKEN),
)
except (RemoteError, NotERC20Conformant) as e:
log.debug(f'Skipping {checksumed_address} due to {str(e)}')
continue
if own_token is not None:
tokens_to_ignore.add(own_token)
return tokens_to_ignore
