def _evaluate_balance_sheet_input(other: Any, operation: str) -> BalanceSheet:
transformed_input = other
if isinstance(other, dict):
if len(other) == 2 and 'assets' in other and 'liabilities' in other:
try:
assets = defaultdict(Balance)
liabilities = defaultdict(Balance)
for asset, entry in other['assets'].items():
assets[asset] = _evaluate_balance_input(entry, operation)
for asset, entry in other['liabilities'].items():
liabilities[asset] = _evaluate_balance_input(
entry, operation)
except InputError as e:
raise InputError(
f'Found valid dict object but with invalid values '
f'during BalanceSheet {operation}', ) from e
transformed_input = BalanceSheet(assets=assets,
liabilities=liabilities)
else:
raise InputError(
f'Found invalid dict object during BalanceSheet {operation}')
elif not isinstance(other, BalanceSheet):
raise InputError(
f'Found a {type(other)} object during BalanceSheet {operation}')
return transformed_input
def add_report_data(
self,
report_id: int,
time: Timestamp,
ts_converter: Callable[[Timestamp], str],
event: ProcessedAccountingEvent,
) -> None:
"""Adds a new entry to a transient report for the PnL history in a given time range
May raise:
- DeserializationError if there is a conflict at serialization of the event
- InputError if the event can not be written to the DB. Probably report id does not exist.
"""
cursor = self.db.conn_transient.cursor()
data = event.serialize_for_db(ts_converter)
query = """
INSERT INTO pnl_events(
report_id, timestamp, data
)
VALUES(?, ?, ?);"""
try:
cursor.execute(query, (report_id, time, data))
except sqlcipher.IntegrityError as e: # pylint: disable=no-member
raise InputError(
f'Could not write {event} data to the DB due to {str(e)}. '
f'Probably report {report_id} does not exist?', ) from e
self.db.conn_transient.commit()
def add_report_overview(
self,
report_id: int,
last_processed_timestamp: Timestamp,
processed_actions: int,
total_actions: int,
pnls: PnlTotals,
) -> None:
"""Inserts the report overview data
May raise:
- InputError if the given report id does not exist
"""
cursor = self.db.conn_transient.cursor()
cursor.execute(
'UPDATE pnl_reports SET last_processed_timestamp=?,'
' processed_actions=?, total_actions=? WHERE identifier=?',
(last_processed_timestamp, processed_actions, total_actions,
report_id),
)
if cursor.rowcount != 1:
raise InputError(
f'Could not insert overview for {report_id}. '
f'Report id could not be found in the DB', )
tuples = []
for event_type, entry in pnls.items():
tuples.append(
(report_id, event_type.serialize(), str(entry.taxable),
str(entry.free))) # noqa: E501
cursor.executemany(
'INSERT OR IGNORE INTO pnl_report_totals(report_id, name, taxable_value, free_value) VALUES(?, ?, ?, ?)', # noqa: E501
tuples,
)
self.db.conn_transient.commit()
def _prepare_ens_call_arguments(addr: ChecksumEthAddress) -> List[Any]:
try:
reversed_domain = address_to_reverse_domain(addr)
except (TypeError, ValueError) as e:
raise InputError(f'Address {addr} has incorrect format or type. {str(e)}') from e
normalized_domain_name = normalize_name(reversed_domain)
arguments = [normal_name_to_hash(normalized_domain_name)]
return arguments
def _evaluate_balance_input(other: Any, operation: str) -> Balance:
transformed_input = other
if isinstance(other, dict):
if len(other) == 2 and 'amount' in other and 'usd_value' in other:
try:
amount = FVal(other['amount'])
usd_value = FVal(other['usd_value'])
except (ValueError, KeyError) as e:
raise InputError(
f'Found valid dict object but with invalid values during Balance {operation}',
) from e
transformed_input = Balance(amount=amount, usd_value=usd_value)
else:
raise InputError(
f'Found invalid dict object during Balance {operation}')
elif not isinstance(other, Balance):
raise InputError(
f'Found a {type(other)} object during Balance {operation}')
return transformed_input
def get_report_data(
self,
filter_: ReportDataFilterQuery,
with_limit: bool,
) -> Tuple[List[ProcessedAccountingEvent], int]:
"""Retrieve the event data of a PnL report depending on the given filter
May raise:
- InputError if the report ID does not exist in the DB
"""
cursor = self.db.conn_transient.cursor()
report_id = filter_.report_id
query_result = cursor.execute(
'SELECT COUNT(*) FROM pnl_reports WHERE identifier=?',
(report_id, ),
)
if query_result.fetchone()[0] != 1:
raise InputError(
f'Tried to get PnL events from non existing report with id {report_id}',
)
query, bindings = filter_.prepare()
query = 'SELECT timestamp, data FROM pnl_events ' + query
results = cursor.execute(query, bindings)
records = []
for result in results:
try:
record = ProcessedAccountingEvent.deserialize_from_db(
result[0], result[1])
except DeserializationError as e:
self.db.msg_aggregator.add_error(
f'Error deserializing AccountingEvent from the DB. Skipping it.'
f'Error was: {str(e)}', )
continue
records.append(record)
if filter_.pagination is not None:
no_pagination_filter = deepcopy(filter_)
no_pagination_filter.pagination = None
query, bindings = no_pagination_filter.prepare()
query = 'SELECT COUNT(*) FROM pnl_events ' + query
results = cursor.execute(query, bindings).fetchone()
total_filter_count = results[0]
else:
total_filter_count = len(records)
return _get_reports_or_events_maybe_limit(
entry_type='events',
entries_found=total_filter_count,
entries=records,
with_limit=with_limit,
)
def purge_report_data(self, report_id: int) -> None:
"""Deletes all report data of the given report from the DB
Raises InputError if the report did not exist in the DB.
"""
cursor = self.db.conn_transient.cursor()
cursor.execute('DELETE FROM pnl_reports WHERE identifier=?',
(report_id, ))
if cursor.rowcount != 1:
raise InputError(
f'Could not delete PnL report {report_id} from the DB. Report was not found',
)
self.db.conn.commit()
self.db.update_last_write()
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError(
'Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(
self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}', )
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
def edit_validator(self, validator_index: int, ownership_proportion: FVal) -> None:
"""Edits the ownership proportion for a validator identified by its index.
May raise:
- InputError if we try to edit a non existing validator.
"""
cursor = self.db.conn.cursor()
cursor.execute(
'UPDATE eth2_validators SET ownership_proportion=? WHERE validator_index = ?',
(str(ownership_proportion), validator_index),
)
if cursor.rowcount == 0:
raise InputError(
f'Tried to edit validator with index {validator_index} '
f'that is not in the database',
)
self.db.update_last_write()
def decode_transaction_hashes(
self, ignore_cache: bool, tx_hashes: Optional[List[EVMTxHash]]
) -> List[HistoryBaseEntry]: # noqa: E501
"""Make sure that receipts are pulled + events decoded for the given transaction hashes.
The transaction hashes must exist in the DB at the time of the call
May raise:
- DeserializationError if there is a problem with conacting a remote to get receipts
- RemoteError if there is a problem with contacting a remote to get receipts
- InputError if the transaction hash is not found in the DB
"""
events = []
self.reload_from_db()
# If no transaction hashes are passed, decode all transactions.
if tx_hashes is None:
tx_hashes = []
cursor = self.database.conn.cursor()
for entry in cursor.execute(
'SELECT tx_hash FROM ethereum_transactions'):
tx_hashes.append(EVMTxHash(entry[0]))
for tx_hash in tx_hashes:
try:
receipt = self.eth_transactions.get_or_query_transaction_receipt(
tx_hash)
except RemoteError as e:
raise InputError(
f'Hash {tx_hash.hex()} does not correspond to a transaction'
) from e # noqa: E501
# TODO: Change this if transaction filter query can accept multiple hashes
txs = self.dbethtx.get_ethereum_transactions(
filter_=ETHTransactionsFilterQuery.make(tx_hash=tx_hash),
has_premium=True, # ignore limiting here
)
events.extend(
self.get_or_decode_transaction_events(
transaction=txs[0],
tx_receipt=receipt,
ignore_cache=ignore_cache,
))
return events
def edit_manually_tracked_balances(db: 'DBHandler',
data: List[ManuallyTrackedBalance]) -> None:
"""Edits manually tracked balances
May raise:
- InputError if the given balances list is empty or if
any of the balance entry labels to edit do not exist in the DB.
- TagConstraintError if any of the given balance data contain unknown tags.
"""
if len(data) == 0:
raise InputError(
'Empty list of manually tracked balances to edit was given')
db.ensure_tags_exist(
given_data=data,
action='editing',
data_type='manually tracked balances',
)
db.edit_manually_tracked_balances(data)
def add_queried_address_for_module(self, module: ModuleName,
address: ChecksumAddress) -> None:
"""May raise:
- InputError: If the address is already in the queried addresses for
the module
"""
cursor = self.db.conn.cursor()
try:
cursor.execute(
'INSERT INTO multisettings(name, value) VALUES(?, ?)',
(f'queried_address_{module}', address),
)
except sqlcipher.DatabaseError as e: # pylint: disable=no-member
raise InputError(
f'Address {address} is already in the queried addresses for {module}',
) from e
self.db.conn.commit()
self.db.update_last_write()
def add_manually_tracked_balances(
db: 'DBHandler',
data: List[ManuallyTrackedBalance],
) -> None:
"""Adds manually tracked balances
May raise:
- InputError if any of the given balance entry labels already exist in the DB
- TagConstraintError if any of the given manually tracked balances contain unknown tags.
"""
if len(data) == 0:
raise InputError(
'Empty list of manually tracked balances to add was given')
db.ensure_tags_exist(
given_data=data,
action='adding',
data_type='manually tracked balances',
)
db.add_manually_tracked_balances(data=data)
def remove_queried_address_for_module(
self,
module: ModuleName,
address: ChecksumAddress,
) -> None:
"""May raise:
- InputError: If the address is not in the queried addresses for
the module
"""
cursor = self.db.conn.cursor()
cursor.execute(
'DELETE FROM multisettings WHERE name=? AND value=?;',
(f'queried_address_{module}', address),
)
if cursor.rowcount != 1:
raise InputError(
f'Address {address} is not in the queried addresses for {module}'
)
self.db.conn.commit()
self.db.update_last_write()
def save_all_binance_pairs(
self,
new_pairs: Iterable['BinancePair'],
location: Location,
) -> None:
"""Saves all possible binance pairs into the GlobalDB.
NB: This is not the user-selected binance pairs. This is just a cache.
May raise:
- InputError if there is a DB insertion failure
"""
query = 'INSERT OR IGNORE INTO binance_pairs(pair, base_asset, quote_asset, location) VALUES (?, ?, ?, ?)' # noqa: E501
cursor = self.db.conn.cursor()
try:
cursor.executemany(query,
[pair.serialize_for_db() for pair in new_pairs])
self.db.add_setting_value(
name=f'binance_pairs_queried_at_{location}', value=ts_now())
except sqlite3.IntegrityError as e:
raise InputError(
f'Tried to add a binance pair to the database but failed due to {str(e)}',
) from e
def delete_validator(self, validator_index: Optional[int], public_key: Optional[str]) -> None:
"""Deletes the given validator from the DB. Due to marshmallow here at least one
of the two arguments is not None.
May raise:
- InputError if the given validator to delete does not exist in the DB
"""
cursor = self.db.conn.cursor()
if validator_index is not None:
field = 'validator_index'
input_tuple = (validator_index,)
else: # public key can't be None due to marshmallow
field = 'public_key'
input_tuple = (public_key,) # type: ignore
cursor.execute(f'DELETE FROM eth2_validators WHERE {field} == ?', input_tuple)
if cursor.rowcount != 1:
raise InputError(
f'Tried to delete eth2 validator with {field} '
f'{input_tuple[0]} from the DB but it did not exist',
)
self.db.update_last_write()
def add_validator(
self,
validator_index: Optional[int],
public_key: Optional[Eth2PubKey],
ownership_proportion: FVal,
) -> None:
"""Adds the given validator to the DB. Due to marshmallow here at least
either validator_index or public key is not None.
May raise:
- RemoteError if there is a problem with querying beaconcha.in for more info
- InputError if the validator is already in the DB
"""
valid_index: int
valid_pubkey: Eth2PubKey
dbeth2 = DBEth2(self.database)
if self.premium is None:
tracked_validators = dbeth2.get_validators()
if len(tracked_validators) >= FREE_VALIDATORS_LIMIT:
raise PremiumPermissionError(
f'Adding validator {validator_index} {public_key} would take you '
f'over the free limit of {FREE_VALIDATORS_LIMIT} for tracked validators',
)
if validator_index is not None and public_key is not None:
valid_index = validator_index
valid_pubkey = public_key
if dbeth2.validator_exists(field='validator_index', arg=valid_index):
raise InputError(f'Validator {valid_index} already exists in the DB')
else: # we are missing one of the 2
if validator_index is None:
field = 'public_key'
arg = public_key
else: # we should have valid index
field = 'validator_index'
arg = validator_index # type: ignore
if dbeth2.validator_exists(field=field, arg=arg): # type: ignore
raise InputError(f'Validator {arg} already exists in the DB')
# at this point we gotta query for one of the two
result = self.beaconchain._query(
module='validator',
endpoint=None,
encoded_args=arg, # type: ignore
)
if not isinstance(result, dict):
raise RemoteError(
f'Validator data for {arg} could not be found. Likely invalid validator.')
try:
valid_index = result['validatorindex']
valid_pubkey = Eth2PubKey(result['pubkey'])
except KeyError as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
raise RemoteError(f'Failed to query beaconcha.in for validator data due to: {msg}') from e # noqa: E501
# by now we have a valid index and pubkey. Add to DB
dbeth2.add_validators([
Eth2Validator(
index=valid_index,
public_key=valid_pubkey,
ownership_proportion=ownership_proportion,
),
])
def import_assets_from_file(
path: Path,
msg_aggregator: 'MessagesAggregator',
db_handler: 'DBHandler',
) -> None:
"""
Import assets from the file at the defined path.
This function can raise:
- ValidationError: If the format of the file is not correct
- InputError: If the version of the file is not valid for the current
globaldb version
"""
globaldb = GlobalDBHandler()
with open(path) as f:
data = ExportedAssetsSchema().loads(f.read())
if int(data['version']) != GLOBAL_DB_VERSION:
raise InputError(
f'Provided file is for a different version of rotki. File version: '
f'{data["version"]} rotki version: {GLOBAL_DB_VERSION}',
)
if data['assets'] is None:
raise InputError('The imported file is missing a valid list of assets')
identifiers = []
for asset_data in data['assets']:
# Check if we already have the asset with that name and symbol. It is possible that
# we have added a missing asset. Using check_asset_exists for non ethereum tokens and
# for ethereum tokens comparing by identifier. The edge case of a non-ethereum token
# with same name and symbol will make this fail.
asset_type = asset_data['asset_type']
asset_ref: Union[Optional[List[str]], Optional[AssetData]]
if asset_type == AssetType.ETHEREUM_TOKEN:
asset_ref = globaldb.get_asset_data(
identifier=asset_data['identifier'],
form_with_incomplete_data=True,
)
else:
asset_ref = globaldb.check_asset_exists(
asset_type=asset_type,
name=asset_data['name'],
symbol=asset_data['symbol'],
)
if asset_ref is not None:
msg_aggregator.add_warning(
f'Tried to import existing asset {asset_data["identifier"]} with '
f'name {asset_data["name"]}',
)
continue
try:
globaldb.add_asset(
asset_id=asset_data['identifier'],
asset_type=asset_type,
data=asset_data['extra_information'],
)
except InputError as e:
log.error(
f'Failed to import asset with {asset_data["identifier"]=}',
f'{asset_type=} and {asset_data=}. {str(e)}',
)
msg_aggregator.add_error(
f'Failed to save import with identifier '
f'{asset_data["identifier"]}. Check logs for more details',
)
continue
identifiers.append(asset_data['identifier'])
db_handler.add_asset_identifiers(identifiers)
def _ens_lookup(
self,
web3: Optional[Web3],
name: str,
blockchain: SupportedBlockchain = SupportedBlockchain.ETHEREUM,
) -> Optional[Union[ChecksumEthAddress, HexStr]]:
"""Performs an ENS lookup and returns address if found else None
TODO: currently web3.py 5.15.0 does not support multichain ENS domains
(EIP-2304), therefore requesting a non-Ethereum address won't use the
web3 ens library and will require to extend the library resolver ABI.
An issue in their repo (#1839) reporting the lack of support has been
created. This function will require refactoring once they include
support for EIP-2304.
https://github.com/ethereum/web3.py/issues/1839
May raise:
- RemoteError if Etherscan is used and there is a problem querying it or
parsing its response
- InputError if the given name is not a valid ENS name
"""
try:
normal_name = normalize_name(name)
except InvalidName as e:
raise InputError(str(e)) from e
resolver_addr = self._call_contract(
web3=web3,
contract_address=ENS_MAINNET_ADDR,
abi=ENS_ABI,
method_name='resolver',
arguments=[normal_name_to_hash(normal_name)],
)
if is_none_or_zero_address(resolver_addr):
return None
ens_resolver_abi = ENS_RESOLVER_ABI.copy()
arguments = [normal_name_to_hash(normal_name)]
if blockchain != SupportedBlockchain.ETHEREUM:
ens_resolver_abi.extend(ENS_RESOLVER_ABI_MULTICHAIN_ADDRESS)
arguments.append(blockchain.ens_coin_type())
try:
deserialized_resolver_addr = deserialize_ethereum_address(resolver_addr)
except DeserializationError:
log.error(
f'Error deserializing address {resolver_addr} while doing'
f'ens lookup',
)
return None
address = self._call_contract(
web3=web3,
contract_address=deserialized_resolver_addr,
abi=ens_resolver_abi,
method_name='addr',
arguments=arguments,
)
if is_none_or_zero_address(address):
return None
if blockchain != SupportedBlockchain.ETHEREUM:
return HexStr(address.hex())
try:
return deserialize_ethereum_address(address)
except DeserializationError:
log.error(f'Error deserializing address {address}')
return None
