def maybe_add_external_trades_to_history(
db: DBHandler,
start_ts: Timestamp,
end_ts: Timestamp,
history: List[Trade],
msg_aggregator: MessagesAggregator,
) -> List[Trade]:
"""
Queries the DB to get any external trades, adds them to the provided history and returns it.
If there is an unexpected error at the external trade deserialization an error is logged.
"""
serialized_external_trades = db.get_trades()
try:
external_trades = trades_from_dictlist(
given_trades=serialized_external_trades,
start_ts=start_ts,
end_ts=end_ts,
location='external trades',
msg_aggregator=msg_aggregator,
)
except KeyError:
msg_aggregator.add_error(
'External trades in the DB are in an unrecognized format')
return history
history.extend(external_trades)
history.sort(key=lambda trade: trade.timestamp)
return history
def maybe_add_external_trades_to_history(
db: DBHandler,
start_ts: Timestamp,
end_ts: Timestamp,
history: List[Union[Trade, MarginPosition]],
msg_aggregator: MessagesAggregator,
) -> List[Union[Trade, MarginPosition]]:
"""
Queries the DB to get any external trades, adds them to the provided history and returns it.
If there is an unexpected error at the external trade deserialization an error is logged.
"""
serialized_external_trades = db.get_trades()
try:
external_trades = trades_from_dictlist(
given_trades=serialized_external_trades,
start_ts=start_ts,
end_ts=end_ts,
location='external trades',
msg_aggregator=msg_aggregator,
)
except (KeyError, DeserializationError):
msg_aggregator.add_error(
'External trades in the DB are in an unrecognized format')
return history
history.extend(external_trades)
# TODO: We also sort in one other place in this file and also in accountant.py
# Get rid of the unneeded cases?
history.sort(key=lambda trade: action_get_timestamp(trade))
return history
def process_polo_loans(
msg_aggregator: MessagesAggregator,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Loan]:
"""Takes in the list of loans from poloniex as returned by the return_lending_history
api call, processes it and returns it into our loan format
"""
new_data = []
for loan in reversed(data):
log.debug('processing poloniex loan', **make_sensitive(loan))
try:
close_time = deserialize_timestamp_from_poloniex_date(
loan['close'])
open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
if open_time < start_ts:
continue
if close_time > end_ts:
continue
our_loan = Loan(
location=Location.POLONIEX,
open_time=open_time,
close_time=close_time,
currency=asset_from_poloniex(loan['currency']),
fee=deserialize_fee(loan['fee']),
earned=deserialize_asset_amount(loan['earned']),
amount_lent=deserialize_asset_amount(loan['amount']),
)
except UnsupportedAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
msg_aggregator.add_error(
'Deserialization error while reading a poloniex loan. Check '
'logs for more details. Ignoring it.', )
log.error(
'Deserialization error while reading a poloniex loan',
loan=loan,
error=msg,
)
continue
new_data.append(our_loan)
new_data.sort(key=lambda loan: loan.open_time)
return new_data
def query_zerion_address(
ethereum: 'EthereumManager',
msg_aggregator: MessagesAggregator,
) -> ChecksumEthAddress:
"""Queries the zerion contract address. If query fails, then last known
address is used"""
result = ethereum.ens_lookup('api.zerion.eth')
if result is None:
msg_aggregator.add_error(
'Could not query api.zerion.eth address. Using last known address',
)
contract_address = ZERION_ADAPTER_ADDRESS
else:
contract_address = result
return contract_address
def query_usd_price_zero_if_error(
asset: Asset,
time: Timestamp,
location: str,
msg_aggregator: MessagesAggregator,
) -> Price:
try:
usd_price = PriceHistorian().query_historical_price(
from_asset=asset,
to_asset=A_USD,
timestamp=time,
)
except (RemoteError, NoPriceForGivenTimestamp):
msg_aggregator.add_error(
f'Could not query usd price for {asset.identifier} and time {time} '
f'when processing {location}. Using zero price', )
usd_price = Price(ZERO)
return usd_price
class TaskManager():
def __init__(
self,
max_tasks_num: int,
greenlet_manager: GreenletManager,
api_task_greenlets: List[gevent.Greenlet],
database: DBHandler,
cryptocompare: Cryptocompare,
premium_sync_manager: Optional[PremiumSyncManager],
chain_manager: ChainManager,
exchange_manager: ExchangeManager,
evm_tx_decoder: 'EVMTransactionDecoder',
eth_transactions: 'EthTransactions',
deactivate_premium: Callable,
query_balances: Callable,
) -> None:
self.max_tasks_num = max_tasks_num
self.greenlet_manager = greenlet_manager
self.api_task_greenlets = api_task_greenlets
self.database = database
self.cryptocompare = cryptocompare
self.exchange_manager = exchange_manager
self.evm_tx_decoder = evm_tx_decoder
self.eth_transactions = eth_transactions
self.cryptocompare_queries: Set[CCHistoQuery] = set()
self.chain_manager = chain_manager
self.last_xpub_derivation_ts = 0
self.last_eth_tx_query_ts: DefaultDict[ChecksumEthAddress,
int] = defaultdict(int)
self.last_exchange_query_ts: DefaultDict[ExchangeLocationID,
int] = defaultdict(int)
self.base_entries_ignore_set: Set[str] = set()
self.prepared_cryptocompare_query = False
self.greenlet_manager.spawn_and_track( # Needs to run in greenlet, is slow
after_seconds=None,
task_name='Prepare cryptocompare queries',
exception_is_error=True,
method=self._prepare_cryptocompare_queries,
)
self.deactivate_premium = deactivate_premium
self.query_balances = query_balances
self.last_premium_status_check = ts_now()
self.msg_aggregator = MessagesAggregator()
self.potential_tasks = [
self._maybe_schedule_cryptocompare_query,
self._maybe_schedule_xpub_derivation,
self._maybe_query_ethereum_transactions,
self._maybe_schedule_exchange_history_query,
self._maybe_schedule_ethereum_txreceipts,
self._maybe_query_missing_prices,
self._maybe_decode_evm_transactions,
self._maybe_check_premium_status,
self._maybe_update_snapshot_balances,
]
if premium_sync_manager is not None:
self.potential_tasks.append(
premium_sync_manager.maybe_upload_data_to_server)
self.schedule_lock = gevent.lock.Semaphore()
def _prepare_cryptocompare_queries(self) -> None:
"""Prepare the queries to do to cryptocompare
This would be really slow if the entire json cache files were read but we
have implemented get_cached_data_metadata to only read the relevant part of the file.
Before doing that we had to yield with gevent.sleep() at each loop iteration.
Runs only once in the beginning and then has a number of queries prepared
for the task manager to schedule
"""
now_ts = ts_now()
if self.prepared_cryptocompare_query is True:
return
if len(self.cryptocompare_queries) != 0:
return
assets = self.database.query_owned_assets()
main_currency = self.database.get_main_currency()
for asset in assets:
if asset.is_fiat() and main_currency.is_fiat():
continue # ignore fiat to fiat
if asset.cryptocompare == '' or main_currency.cryptocompare == '':
continue # not supported in cryptocompare
if asset.cryptocompare is None and asset.symbol is None:
continue # type: ignore # asset.symbol may be None for auto generated underlying tokens # noqa: E501
data_range = GlobalDBHandler().get_historical_price_range(
from_asset=asset,
to_asset=main_currency,
source=HistoricalPriceOracle.CRYPTOCOMPARE,
)
if data_range is not None and now_ts - data_range[
1] < CRYPTOCOMPARE_QUERY_AFTER_SECS:
continue
self.cryptocompare_queries.add(
CCHistoQuery(from_asset=asset, to_asset=main_currency))
self.prepared_cryptocompare_query = True
def _maybe_schedule_cryptocompare_query(self) -> bool:
"""Schedules a cryptocompare query for a single asset history"""
if self.prepared_cryptocompare_query is False:
return False
if len(self.cryptocompare_queries) == 0:
return False
# If there is already a cryptocompary query running don't schedule another
if any('Cryptocompare historical prices' in x.task_name
for x in self.greenlet_manager.greenlets):
return False
now_ts = ts_now()
# Make sure there is a long enough period between an asset's histohour query
# to avoid getting rate limited by cryptocompare
if now_ts - self.cryptocompare.last_histohour_query_ts <= CRYPTOCOMPARE_HISTOHOUR_FREQUENCY: # noqa: E501
return False
query = self.cryptocompare_queries.pop()
task_name = f'Cryptocompare historical prices {query.from_asset} / {query.to_asset} query'
log.debug(f'Scheduling task for {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=False,
method=self.cryptocompare.query_and_store_historical_data,
from_asset=query.from_asset,
to_asset=query.to_asset,
timestamp=now_ts,
)
return True
def _maybe_schedule_xpub_derivation(self) -> None:
"""Schedules the xpub derivation task if enough time has passed and if user has xpubs"""
now = ts_now()
if now - self.last_xpub_derivation_ts <= XPUB_DERIVATION_FREQUENCY:
return
xpubs = self.database.get_bitcoin_xpub_data()
if len(xpubs) == 0:
return
log.debug('Scheduling task for Xpub derivation')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='Derive new xpub addresses',
exception_is_error=True,
method=XpubManager(
self.chain_manager).check_for_new_xpub_addresses,
)
self.last_xpub_derivation_ts = now
def _maybe_query_ethereum_transactions(self) -> None:
"""Schedules the ethereum transaction query task if enough time has passed"""
accounts = self.database.get_blockchain_accounts().eth
if len(accounts) == 0:
return
now = ts_now()
dbethtx = DBEthTx(self.database)
queriable_accounts = []
for account in accounts:
_, end_ts = dbethtx.get_queried_range(account)
if now - max(self.last_eth_tx_query_ts[account],
end_ts) > ETH_TX_QUERY_FREQUENCY:
queriable_accounts.append(account)
if len(queriable_accounts) == 0:
return
address = random.choice(queriable_accounts)
task_name = f'Query ethereum transactions for {address}'
log.debug(f'Scheduling task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=self.eth_transactions.single_address_query_transactions,
address=address,
start_ts=0,
end_ts=now,
)
self.last_eth_tx_query_ts[address] = now
def _maybe_schedule_ethereum_txreceipts(self) -> None:
"""Schedules the ethereum transaction receipts query task
The DB check happens first here to see if scheduling would even be needed.
But the DB query will happen again inside the query task while having the
lock acquired.
"""
dbethtx = DBEthTx(self.database)
hash_results = dbethtx.get_transaction_hashes_no_receipt(
tx_filter_query=None, limit=TX_RECEIPTS_QUERY_LIMIT) # noqa: E501
if len(hash_results) == 0:
return
task_name = f'Query {len(hash_results)} ethereum transactions receipts'
log.debug(f'Scheduling task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=self.eth_transactions.
get_receipts_for_transactions_missing_them,
limit=TX_RECEIPTS_QUERY_LIMIT,
)
def _maybe_schedule_exchange_history_query(self) -> None:
"""Schedules the exchange history query task if enough time has passed"""
if len(self.exchange_manager.connected_exchanges) == 0:
return
now = ts_now()
queriable_exchanges = []
for exchange in self.exchange_manager.iterate_exchanges():
if exchange.location in (Location.BINANCE, Location.BINANCEUS):
continue # skip binance due to the way their history is queried and rate limiting
queried_range = self.database.get_used_query_range(
f'{str(exchange.location)}_trades')
end_ts = queried_range[1] if queried_range else 0
if now - max(self.last_exchange_query_ts[exchange.location_id()],
end_ts) > EXCHANGE_QUERY_FREQUENCY: # noqa: E501
queriable_exchanges.append(exchange)
if len(queriable_exchanges) == 0:
return
exchange = random.choice(queriable_exchanges)
task_name = f'Query history of {exchange.name} exchange'
log.debug(f'Scheduling task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=exchange.query_history_with_callbacks,
start_ts=0,
end_ts=now,
success_callback=noop_exchange_success_cb,
fail_callback=exchange_fail_cb,
)
self.last_exchange_query_ts[exchange.location_id()] = now
def _maybe_query_missing_prices(self) -> None:
query_filter = HistoryEventFilterQuery.make(limit=100)
entries = self.get_base_entries_missing_prices(query_filter)
if len(entries) > 0:
task_name = 'Periodically query history events prices'
log.debug(f'Scheduling task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=self.query_missing_prices_of_base_entries,
entries_missing_prices=entries,
)
def get_base_entries_missing_prices(
self,
query_filter: HistoryEventFilterQuery,
) -> List[Tuple[str, FVal, Asset, Timestamp]]:
"""
Searches base entries missing usd prices that have not previously been checked in
this session.
"""
# Use a deepcopy to avoid mutations in the filter query if it is used later
db = DBHistoryEvents(self.database)
new_query_filter = copy.deepcopy(query_filter)
new_query_filter.filters.append(
DBStringFilter(and_op=True, column='usd_value', value='0'), )
new_query_filter.filters.append(
DBIgnoreValuesFilter(
and_op=True,
column='identifier',
values=list(self.base_entries_ignore_set),
), )
return db.rows_missing_prices_in_base_entries(
filter_query=new_query_filter)
def query_missing_prices_of_base_entries(
self,
entries_missing_prices: List[Tuple[str, FVal, Asset, Timestamp]],
) -> None:
"""Queries missing prices for HistoryBaseEntry in database updating
the price if it is found. Otherwise we add the id to the ignore list
for this session.
"""
inquirer = PriceHistorian()
updates = []
for identifier, amount, asset, timestamp in entries_missing_prices:
try:
price = inquirer.query_historical_price(
from_asset=asset,
to_asset=A_USD,
timestamp=timestamp,
)
except (NoPriceForGivenTimestamp, RemoteError) as e:
log.error(
f'Failed to find price for {asset} at {timestamp} in base '
f'entry {identifier}. {str(e)}.', )
self.base_entries_ignore_set.add(identifier)
continue
usd_value = amount * price
updates.append((str(usd_value), identifier))
query = 'UPDATE history_events SET usd_value=? WHERE rowid=?'
cursor = self.database.conn.cursor()
cursor.executemany(query, updates)
self.database.update_last_write()
def _maybe_decode_evm_transactions(self) -> None:
"""Schedules the evm transaction decoding task
The DB check happens first here to see if scheduling would even be needed.
But the DB query will happen again inside the query task while having the
lock acquired.
"""
dbethtx = DBEthTx(self.database)
hashes = dbethtx.get_transaction_hashes_not_decoded(
limit=TX_DECODING_LIMIT)
hashes_length = len(hashes)
if hashes_length > 0:
task_name = f'decode {hashes_length} evm trasactions'
log.debug(f'Scheduling periodic task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=self.evm_tx_decoder.
get_and_decode_undecoded_transactions,
limit=TX_DECODING_LIMIT,
)
def _maybe_check_premium_status(self) -> None:
"""
Validates the premium status of the account and if the credentials are not valid
it deactivates the user's premium status.
"""
now = ts_now()
if now - self.last_premium_status_check < PREMIUM_STATUS_CHECK:
return
db_credentials = self.database.get_rotkehlchen_premium()
if db_credentials:
try:
premium_create_and_verify(db_credentials)
except PremiumAuthenticationError as e:
message = (
f'Could not authenticate with the rotkehlchen server with '
f'the API keys found in the Database. Error: {str(e)}. Will '
f'deactivate the premium status.')
self.msg_aggregator.add_error(message)
self.deactivate_premium()
self.last_premium_status_check = now
def _maybe_update_snapshot_balances(self) -> None:
"""
Update the balances of a user if the difference between last time they were updated
and the current time exceeds the `balance_save_frequency`.
"""
if self.database.should_save_balances():
task_name = 'Periodically update snapshot balances'
log.debug(f'Scheduling task to {task_name}')
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name=task_name,
exception_is_error=True,
method=self.query_balances,
requested_save_data=True,
save_despite_errors=False,
timestamp=None,
ignore_cache=True,
)
def _schedule(self) -> None:
"""Schedules background tasks"""
self.greenlet_manager.clear_finished()
current_greenlets = len(self.greenlet_manager.greenlets) + len(
self.api_task_greenlets)
not_proceed = current_greenlets >= self.max_tasks_num
log.debug(
f'At task scheduling. Current greenlets: {current_greenlets} '
f'Max greenlets: {self.max_tasks_num}. '
f'{"Will not schedule" if not_proceed else "Will schedule"}.', )
if not_proceed:
return # too busy
callables = random.sample(
population=self.potential_tasks,
k=min(self.max_tasks_num - current_greenlets,
len(self.potential_tasks)),
)
for callable_fn in callables:
callable_fn()
def schedule(self) -> None:
"""Schedules background task while holding the scheduling lock
Used during logout to make sure no task is being scheduled at the same time
as logging out
"""
with self.schedule_lock:
self._schedule()
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator)
# Initialize the AssetResolver singleton
AssetResolver(data_directory=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko)
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='periodically_query_icons_until_all_cached',
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.lock.release()
self.shutdown_event = gevent.event.Event()
def reset_after_failed_account_creation_or_login(self) -> None:
"""If the account creation or login failed make sure that the Rotki instance is clear
Tricky instances are when after either failed premium credentials or user refusal
to sync premium databases we relogged in.
"""
self.cryptocompare.db = None
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new, initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_error(
'Tried to synchronize the database from remote but the local password '
'does not match the one the remote DB has. Please change the password '
'to be the same as the password of the account you want to sync from ',
)
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
method=maybe_submit_usage_analytics,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator)
historical_data_start = settings.historical_data_start
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=self.cryptocompare,
)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
database=self.data.db,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
Inquirer().inject_ethereum(ethereum_manager)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
)
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
self.greenlet_manager.clear()
del self.chain_manager
self.exchange_manager.delete_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
del self.data_importer
if self.premium is not None:
del self.premium
self.data.logout()
self.password = ''
self.cryptocompare.unset_database()
# Make sure no messages leak to other user sessions
self.msg_aggregator.consume_errors()
self.msg_aggregator.consume_warnings()
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, credentials: PremiumCredentials) -> None:
"""
Sets the premium credentials for Rotki
Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(credentials)
else:
self.premium = premium_create_and_verify(credentials)
self.data.db.set_rotkehlchen_premium(credentials)
def delete_premium_credentials(self) -> Tuple[bool, str]:
"""Deletes the premium credentials for Rotki"""
msg = ''
success = self.data.db.del_rotkehlchen_premium()
if success is False:
msg = 'The database was unable to delete the Premium keys for the logged-in user'
self.deactivate_premium_status()
return success, msg
def deactivate_premium_status(self) -> None:
"""Deactivate premium in the current session"""
self.premium = None
self.premium_sync_manager.premium = None
self.chain_manager.deactivate_premium_status()
def start(self) -> gevent.Greenlet:
return gevent.spawn(self.main_loop)
def main_loop(self) -> None:
"""Rotki main loop that fires often and manages many different tasks
Each task remembers the last time it run successfully and know how often it
should run. So each task manages itself.
"""
# super hacky -- organize better when recurring tasks are implemented
# https://github.com/rotki/rotki/issues/1106
xpub_derivation_scheduled = False
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
if self.user_is_logged_in:
log.debug('Main loop start')
self.premium_sync_manager.maybe_upload_data_to_server()
if not xpub_derivation_scheduled:
# 1 minute in the app's startup try to derive new xpub addresses
self.greenlet_manager.spawn_and_track(
after_seconds=60.0,
task_name='Derive new xpub addresses',
method=XpubManager(self.chain_manager).check_for_new_xpub_addresses,
)
xpub_derivation_scheduled = True
log.debug('Main loop end')
def get_blockchain_account_data(
self,
blockchain: SupportedBlockchain,
) -> Union[List[BlockchainAccountData], Dict[str, Any]]:
account_data = self.data.db.get_blockchain_account_data(blockchain)
if blockchain != SupportedBlockchain.BITCOIN:
return account_data
xpub_data = self.data.db.get_bitcoin_xpub_data()
addresses_to_account_data = {x.address: x for x in account_data}
address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping(
list(addresses_to_account_data.keys()), # type: ignore
)
xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {}
for address, xpub_entry in address_to_xpub_mappings.items():
if xpub_entry not in xpub_mappings:
xpub_mappings[xpub_entry] = []
xpub_mappings[xpub_entry].append(addresses_to_account_data[address])
data: Dict[str, Any] = {'standalone': [], 'xpubs': []}
# Add xpub data
for xpub_entry in xpub_data:
data_entry = xpub_entry.serialize()
addresses = xpub_mappings.get(xpub_entry, None)
data_entry['addresses'] = addresses if addresses and len(addresses) != 0 else None
data['xpubs'].append(data_entry)
# Add standalone addresses
for account in account_data:
if account.address not in address_to_xpub_mappings:
data['standalone'].append(account)
return data
def add_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> BlockchainBalancesUpdate:
"""Adds new blockchain accounts
Adds the accounts to the blockchain instance and queries them to get the
updated balances. Also adds them in the DB
May raise:
- EthSyncError from modify_blockchain_account
- InputError if the given accounts list is empty.
- TagConstraintError if any of the given account data contain unknown tags.
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
"""
self.data.db.ensure_tags_exist(
given_data=account_data,
action='adding',
data_type='blockchain accounts',
)
address_type = blockchain.get_address_type()
updated_balances = self.chain_manager.add_blockchain_accounts(
blockchain=blockchain,
accounts=[address_type(entry.address) for entry in account_data],
)
self.data.db.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return updated_balances
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError('Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}',
)
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return None
def remove_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
) -> BlockchainBalancesUpdate:
"""Removes blockchain accounts
Removes the accounts from the blockchain instance and queries them to get
the updated balances. Also removes them from the DB
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- InputError if a non-existing account was given to remove
"""
balances_update = self.chain_manager.remove_blockchain_accounts(
blockchain=blockchain,
accounts=accounts,
)
self.data.db.remove_blockchain_accounts(blockchain, accounts)
return balances_update
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
defi_events,
) = self.trades_historian.get_history(
start_ts=start_ts,
end_ts=end_ts,
has_premium=self.premium is not None,
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
defi_events=defi_events,
)
return result, error_or_empty
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade'],
location_actions: TRADES_LIST,
all_actions: TRADES_LIST,
) -> TRADES_LIST:
...
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['asset_movement'],
location_actions: List[AssetMovement],
all_actions: List[AssetMovement],
) -> List[AssetMovement]:
...
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade', 'asset_movement'],
location_actions: Union[TRADES_LIST, List[AssetMovement]],
all_actions: Union[TRADES_LIST, List[AssetMovement]],
) -> Union[TRADES_LIST, List[AssetMovement]]:
"""Take as many actions from location actions and add them to all actions as the limit permits
Returns the modified (or not) all_actions
"""
# If we are already at or above the limit return current actions disregarding this location
actions_mapping = self.actions_per_location[action_type]
current_num_actions = sum(x for _, x in actions_mapping.items())
limit = LIMITS_MAPPING[action_type]
if current_num_actions >= limit:
return all_actions
# Find out how many more actions can we return, and depending on that get
# the number of actions from the location actions and add them to the total
remaining_num_actions = limit - current_num_actions
if remaining_num_actions < 0:
remaining_num_actions = 0
num_actions_to_take = min(len(location_actions), remaining_num_actions)
actions_mapping[location] = num_actions_to_take
all_actions.extend(location_actions[0:num_actions_to_take]) # type: ignore
return all_actions
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> TRADES_LIST:
"""Queries trades for the given location and time range.
If no location is given then all external, all exchange and DEX trades are queried.
DEX Trades are queried only if the user has premium
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
trades: TRADES_LIST
if location is not None:
trades = self.query_location_trades(from_ts, to_ts, location)
else:
trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL)
# crypto.com is not an API key supported exchange but user can import from CSV
trades.extend(self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM))
for name, exchange in self.exchange_manager.connected_exchanges.items():
exchange_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts)
if self.premium is None:
trades = self._apply_actions_limit(
location=deserialize_location(name),
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# for all trades we also need uniswap trades
if self.premium is not None:
uniswap = self.chain_manager.uniswap
if uniswap is not None:
trades.extend(
uniswap.get_trades(
addresses=self.chain_manager.queried_addresses_for_module('uniswap'),
from_timestamp=from_ts,
to_timestamp=to_ts,
),
)
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
def query_location_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Location,
) -> TRADES_LIST:
# clear the trades queried for this location
self.actions_per_location['trade'][location] = 0
location_trades: TRADES_LIST
if location in (Location.EXTERNAL, Location.CRYPTOCOM):
location_trades = self.data.db.get_trades( # type: ignore # list invariance
from_ts=from_ts,
to_ts=to_ts,
location=location,
)
elif location == Location.UNISWAP:
if self.premium is not None:
uniswap = self.chain_manager.uniswap
if uniswap is not None:
location_trades = uniswap.get_trades( # type: ignore # list invariance
addresses=self.chain_manager.queried_addresses_for_module('uniswap'),
from_timestamp=from_ts,
to_timestamp=to_ts,
)
else:
# should only be an exchange
exchange = self.exchange_manager.get(str(location))
if not exchange:
logger.warn(
f'Tried to query trades from {location} which is either not an '
f'exchange or not an exchange the user has connected to',
)
return []
location_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts)
trades: TRADES_LIST = []
if self.premium is None:
trades = self._apply_actions_limit(
location=location,
action_type='trade',
location_actions=location_trades,
all_actions=trades,
)
else:
trades = location_trades
return trades
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
ignore_cache: bool = False,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are always saved in the DB,
if it is False then data are saved if self.data.should_save_balances()
is True.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
If ignore_cache is True then all underlying calls that have a cache ignore it
Returns a dictionary with the queried balances.
"""
log.info('query_balances called', requested_save_data=requested_save_data)
balances = {}
problem_free = True
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange_balances, _ = exchange.query_balances(ignore_cache=ignore_cache)
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange.name] = exchange_balances
try:
blockchain_result = self.chain_manager.query_balances(
blockchain=None,
force_token_detection=ignore_cache,
ignore_cache=ignore_cache,
)
serialized_chain_result = blockchain_result.totals.to_dict()
balances['blockchain'] = serialized_chain_result['assets']
except (RemoteError, EthSyncError) as e:
problem_free = False
log.error(f'Querying blockchain balances failed due to: {str(e)}')
balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances)
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()]
liabilities = serialized_chain_result['liabilities'] # atm liabilities only on chain
# calculate net usd value
net_usd = ZERO
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
# subtract liabilities
liabilities_total_usd = sum(x['usd_value'] for _, x in liabilities.items())
net_usd -= liabilities_total_usd
stats: Dict[str, Any] = {
'location': {
},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if name == 'blockchain': # blockchain is the only location with liabilities atm
total -= liabilities_total_usd
if net_usd != ZERO:
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != ZERO:
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
for k, v in liabilities.items():
if net_usd != ZERO:
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
liabilities[k]['percentage_of_net_value'] = percentage
balance_sheet = {
'assets': combined,
'liabilities': liabilities,
}
result_dict = merge_dicts(balance_sheet, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances()
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.db.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
return result_dict
def _query_exchange_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
all_movements: List[AssetMovement],
exchange: Union[ExchangeInterface, Location],
) -> List[AssetMovement]:
if isinstance(exchange, ExchangeInterface):
location = deserialize_location(exchange.name)
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
location_movements = exchange.query_deposits_withdrawals(
start_ts=from_ts,
end_ts=to_ts,
)
else:
assert isinstance(exchange, Location), 'only a location should make it here'
assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here'
location = exchange
# cryptocom has no exchange integration but we may have DB entries
self.actions_per_location['asset_movement'][location] = 0
location_movements = self.data.db.get_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
location=str(location),
)
movements: List[AssetMovement] = []
if self.premium is None:
movements = self._apply_actions_limit(
location=location,
action_type='asset_movement',
location_actions=location_movements,
all_actions=all_movements,
)
else:
all_movements.extend(location_movements)
movements = all_movements
return movements
def query_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> List[AssetMovement]:
"""Queries AssetMovements for the given location and time range.
If no location is given then all exchange asset movements are queried.
If the user does not have premium then a limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
movements: List[AssetMovement] = []
if location is not None:
if location == Location.CRYPTOCOM:
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=Location.CRYPTOCOM,
)
else:
exchange = self.exchange_manager.get(str(location))
if not exchange:
logger.warn(
f'Tried to query deposits/withdrawals from {location} which is either '
f'not at exchange or not an exchange the user has connected to',
)
return []
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
)
else:
# cryptocom has no exchange integration but we may have DB entries due to csv import
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=Location.CRYPTOCOM,
)
for _, exchange in self.exchange_manager.connected_exchanges.items():
self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
)
# return movements with most recent first
movements.sort(key=lambda x: x.timestamp, reverse=True)
return movements
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
with self.lock:
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.kraken_account_type is not None:
kraken = self.exchange_manager.get('kraken')
if kraken:
kraken.set_account_type(settings.kraken_account_type) # type: ignore
self.data.db.set_settings(settings)
return True, ''
def get_settings(self) -> DBSettings:
"""Returns the db settings with a check whether premium is active or not"""
db_settings = self.data.db.get_settings(have_premium=self.premium is not None)
return db_settings
def setup_exchange(
self,
name: str,
api_key: ApiKey,
api_secret: ApiSecret,
passphrase: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret and optionally a passphrase
By default the api keys are always validated unless validate is False.
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
passphrase=passphrase,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase)
return is_success, msg
def remove_exchange(self, name: str) -> Tuple[bool, str]:
if not self.exchange_manager.has_exchange(name):
return False, 'Exchange {} is not registered'.format(name)
self.exchange_manager.delete_exchange(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
self.data.db.delete_used_query_range_for_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result['last_balance_save'] = self.data.db.get_last_balance_save_time()
result['eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(NodeName.OWN, None) is not None # noqa : E501
result['history_process_start_ts'] = self.accountant.started_processing_timestamp
result['history_process_current_ts'] = self.accountant.currently_processing_timestamp
result['last_data_upload_ts'] = Timestamp(self.premium_sync_manager.last_data_upload_ts) # noqa : E501
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
def history_event_from_kraken(
events: List[Dict[str, Any]],
name: str,
msg_aggregator: MessagesAggregator,
) -> Tuple[List[HistoryBaseEntry], bool]:
"""
This function gets raw data from kraken and creates a list of related history events
to be used in the app. It returns a list of events and a boolean in the case that an unknown
type is found.
"""
group_events = []
found_unknown_event = False
current_fee_index = len(events)
for idx, raw_event in enumerate(events):
try:
timestamp = TimestampMS((deserialize_fval(
value=raw_event['time'],
name='time',
location='kraken ledger processing',
) * 1000).to_int(exact=False))
identifier = raw_event['refid']
event_type = kraken_ledger_entry_type_to_ours(raw_event['type'])
asset = asset_from_kraken(raw_event['asset'])
event_subtype = HistoryEventSubType.NONE
notes = None
raw_amount = deserialize_asset_amount(raw_event['amount'])
# If we don't know how to handle an event atm or we find an unsupported
# event type the logic will be to store it as unknown and if in the future
# we need some information from it we can take actions to process them
if event_type == HistoryEventType.TRANSFER:
if raw_event['subtype'] == 'spottostaking':
event_type = HistoryEventType.STAKING
event_subtype = HistoryEventSubType.DEPOSIT_ASSET
elif raw_event['subtype'] == 'stakingfromspot':
event_type = HistoryEventType.STAKING
event_subtype = HistoryEventSubType.RECEIVE_WRAPPED
elif raw_event['subtype'] == 'stakingtospot':
event_type = HistoryEventType.STAKING
event_subtype = HistoryEventSubType.REMOVE_ASSET
elif raw_event['subtype'] == 'spotfromstaking':
event_type = HistoryEventType.STAKING
event_subtype = HistoryEventSubType.RETURN_WRAPPED
elif event_type == HistoryEventType.ADJUSTMENT:
if raw_amount < ZERO:
event_subtype = HistoryEventSubType.SPEND
else:
event_subtype = HistoryEventSubType.RECEIVE
elif event_type == HistoryEventType.STAKING:
event_subtype = HistoryEventSubType.REWARD
elif event_type == HistoryEventType.INFORMATIONAL:
found_unknown_event = True
notes = raw_event['type']
log.warning(
f'Encountered kraken historic event type we do not process. {raw_event}',
)
fee_amount = deserialize_asset_amount(raw_event['fee'])
# Make sure to not generate an event for KFEES that is not of type FEE
if asset != A_KFEE:
group_events.append(
HistoryBaseEntry(
event_identifier=identifier,
sequence_index=idx,
timestamp=timestamp,
location=Location.KRAKEN,
location_label=name,
asset=asset,
balance=Balance(
amount=raw_amount,
usd_value=ZERO,
),
notes=notes,
event_type=event_type,
event_subtype=event_subtype,
))
if fee_amount != ZERO:
group_events.append(
HistoryBaseEntry(
event_identifier=identifier,
sequence_index=current_fee_index,
timestamp=timestamp,
location=Location.KRAKEN,
location_label=name,
asset=asset,
balance=Balance(
amount=fee_amount,
usd_value=ZERO,
),
notes=notes,
event_type=event_type,
event_subtype=HistoryEventSubType.FEE,
))
# Increase the fee index to not have duplicates in the case of having a normal
# fee and KFEE
current_fee_index += 1
except (DeserializationError, KeyError, UnknownAsset) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Keyrror {msg}'
msg_aggregator.add_error(
f'Failed to read ledger event from kraken {raw_event} due to {msg}',
)
return [], False
return group_events, found_unknown_event
