class Rotkehlchen(object):
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache: typing.ResultCache = dict()
self.premium = None
self.connected_exchanges = []
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.last_data_upload_ts = 0
self.poloniex = None
self.kraken = None
self.bittrex = None
self.bitmex = None
self.binance = None
self.data = DataHandler(self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def initialize_exchanges(self, secret_data):
# initialize exchanges for which we have keys and are not already initialized
if self.kraken is None and 'kraken' in secret_data:
self.kraken = Kraken(
str.encode(secret_data['kraken']['api_key']),
str.encode(secret_data['kraken']['api_secret']), self.data_dir)
self.connected_exchanges.append('kraken')
self.trades_historian.set_exchange('kraken', self.kraken)
if self.poloniex is None and 'poloniex' in secret_data:
self.poloniex = Poloniex(
str.encode(secret_data['poloniex']['api_key']),
str.encode(secret_data['poloniex']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('poloniex')
self.trades_historian.set_exchange('poloniex', self.poloniex)
if self.bittrex is None and 'bittrex' in secret_data:
self.bittrex = Bittrex(
str.encode(secret_data['bittrex']['api_key']),
str.encode(secret_data['bittrex']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('bittrex')
self.trades_historian.set_exchange('bittrex', self.bittrex)
if self.binance is None and 'binance' in secret_data:
self.binance = Binance(
str.encode(secret_data['binance']['api_key']),
str.encode(secret_data['binance']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('binance')
self.trades_historian.set_exchange('binance', self.binance)
if self.bitmex is None and 'bitmex' in secret_data:
self.bitmex = Bitmex(
str.encode(secret_data['bitmex']['api_key']),
str.encode(secret_data['bitmex']['api_secret']), self.inquirer,
self.data_dir)
self.connected_exchanges.append('bitmex')
self.trades_historian.set_exchange('bitmex', self.bitmex)
def remove_all_exchanges(self):
if self.kraken is not None:
self.delete_exchange_data('kraken')
if self.poloniex is not None:
self.delete_exchange_data('poloniex')
if self.bittrex is not None:
self.delete_exchange_data('bittrex')
if self.binance is not None:
self.delete_exchange_data('binance')
if self.bitmex is not None:
self.delete_exchange_data('bitmex')
def try_premium_at_start(self, api_key, api_secret, create_new,
sync_approval, user_dir):
"""Check if new user provided api pair or we already got one in the DB"""
if api_key != '':
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
log.error('Given API key is invalid')
# At this point we are at a new user trying to create an account with
# premium API keys and we failed. But a directory was created. Remove it.
shutil.rmtree(user_dir)
raise AuthenticationError(
'Could not verify keys for the new account. '
'{}'.format(empty_or_error))
else:
# If we got premium initialize it and try to sync with the server
premium_credentials = self.data.db.get_rotkehlchen_premium()
if premium_credentials:
api_key = premium_credentials[0]
api_secret = premium_credentials[1]
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
log.error(
'The API keys found in the Database are not valid. Perhaps '
'they expired?')
del self.premium
self.premium = None
return
else:
# no premium credentials in the DB
return
if self.can_sync_data_from_server():
if sync_approval == 'unknown' and not create_new:
log.info('DB data at server newer than local')
raise PermissionError(
'Rotkehlchen Server has newer version of your DB data. '
'Should we replace local data with the server\'s?')
elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new:
log.info('User approved data sync from server')
if self.sync_data_from_server():
if create_new:
# if we successfully synced data from the server and this is
# a new account, make sure the api keys are properly stored
# in the DB
self.data.db.set_rotkehlchen_premium(
api_key, api_secret)
else:
log.debug('Could sync data from server but user refused')
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
user_dir = self.data.unlock(user, password, create_new)
self.try_premium_at_start(api_key, api_secret, create_new,
sync_approval, user_dir)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
self.data_dir,
self.data.db,
self.data.get_eth_accounts(),
historical_data_start,
)
price_historian = PriceHistorian(
self.data_dir,
historical_data_start,
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
price_historian=price_historian,
profit_currency=self.data.main_currency(),
user_directory=user_dir,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'])
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.inquirer = Inquirer(kraken=self.kraken)
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
all_eth_tokens=self.data.eth_tokens,
owned_eth_tokens=self.data.db.get_owned_tokens(),
inquirer=self.inquirer,
ethchain=ethchain,
)
def logout(self):
user = self.data.username,
log.info(
'Logging out user',
user=user,
)
del self.blockchain
self.blockchain = None
self.remove_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.inquirer
self.inquirer = None
del self.accountant
self.accountant = None
del self.trades_historian
self.trades_historian = None
if self.premium is not None:
del self.premium
self.premium = None
self.data.logout()
self.password = None
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, api_key, api_secret):
log.info('Setting new premium credentials')
if self.premium is not None:
valid, empty_or_error = self.premium.set_credentials(
api_key, api_secret)
else:
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if valid:
self.data.set_premium_credentials(api_key, api_secret)
return True, ''
log.error('Setting new premium credentials failed',
error=empty_or_error)
return False, empty_or_error
def maybe_upload_data_to_server(self):
# upload only if unlocked user has premium
if self.premium is None:
return
# upload only once per hour
diff = ts_now() - self.last_data_upload_ts
if diff > 3600:
self.upload_data_to_server()
def upload_data_to_server(self):
log.debug('upload to server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
log.debug(
'upload to server -- query last metadata failed',
error=result_or_error,
)
return
log.debug(
'CAN_PUSH',
ours=our_hash,
theirs=result_or_error['data_hash'],
)
if our_hash == result_or_error['data_hash']:
log.debug('upload to server -- same hash')
# same hash -- no need to upload anything
return
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts <= result_or_error['last_modify_ts']:
# Server's DB was modified after our local DB
log.debug("CAN_PUSH -> 3")
log.debug('upload to server -- remote db more recent than local')
return
success, result_or_error = self.premium.upload_data(
data, our_hash, our_last_write_ts, 'zlib')
if not success:
log.debug('upload to server -- upload error',
error=result_or_error)
return
self.last_data_upload_ts = ts_now()
log.debug('upload to server -- success')
def can_sync_data_from_server(self):
log.debug('sync data from server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
log.debug('sync data from server failed', error=result_or_error)
return False
log.debug(
'CAN_PULL',
ours=our_hash,
theirs=result_or_error['data_hash'],
)
if our_hash == result_or_error['data_hash']:
log.debug('sync from server -- same hash')
# same hash -- no need to get anything
return False
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts >= result_or_error['last_modify_ts']:
# Local DB is newer than Server DB
log.debug('sync from server -- local DB more recent than remote')
return False
return True
def sync_data_from_server(self):
success, error_or_result = self.premium.pull_data()
if not success:
log.debug('sync from server -- pulling failed.',
error=error_or_result)
return False
self.data.decompress_and_decrypt_db(self.password,
error_or_result['data'])
return True
def start(self):
return gevent.spawn(self.main_loop)
def main_loop(self):
while True and not self.shutdown_event.is_set():
log.debug('Main loop start')
if self.poloniex is not None:
self.poloniex.main_logic()
if self.kraken is not None:
self.kraken.main_logic()
self.maybe_upload_data_to_server()
log.debug('Main loop end')
gevent.sleep(MAIN_LOOP_SECS_DELAY)
def add_blockchain_account(self, blockchain, account):
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(self, blockchain, account):
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens):
try:
new_data = self.blockchain.track_new_tokens(tokens)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens):
try:
new_data = self.blockchain.remove_eth_tokens(tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(self, start_ts, end_ts):
(
error_or_empty, history, margin_history, loan_history,
asset_movements, eth_transactions
) = self.trades_historian.get_history(
start_ts=
0, # For entire history processing we need to have full history available
end_ts=ts_now(),
end_at_least_ts=end_ts)
result = self.accountant.process_history(start_ts, end_ts, history,
margin_history, loan_history,
asset_movements,
eth_transactions)
return result, error_or_empty
def query_fiat_balances(self):
log.info('query_fiat_balances called')
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate
}
return result
def query_balances(self, requested_save_data=False):
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for exchange in self.connected_exchanges:
exchange_balances, msg = getattr(self, exchange).query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not exchange_balances:
problem_free = False
else:
balances[exchange] = exchange_balances
result, error_or_empty = self.blockchain.query_balances()
if error_or_empty == '':
balances['blockchain'] = result['totals']
else:
problem_free = False
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for k, v in combined.items():
net_usd += FVal(v['usd_value'])
stats = {'location': {}, 'net_usd': net_usd}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
self.data.save_balances_data(result_dict)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.data.accountant.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency):
with self.lock:
self.data.set_main_currency(currency, self.accountant)
if currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', currency)
def set_settings(self, settings):
log.info('Add new settings')
message = ''
with self.lock:
if 'eth_rpc_port' in settings:
result, msg = self.blockchain.set_eth_rpc_port(
settings['eth_rpc_port'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_port']
message += "\nEthereum RPC port not set: " + msg
if 'main_currency' in settings:
main_currency = settings['main_currency']
if main_currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def usd_to_main_currency(self, amount):
main_currency = self.data.main_currency()
if main_currency != 'USD' and not hasattr(self,
'usd_to_main_currency_rate'):
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
return self.usd_to_main_currency_rate * amount
def setup_exchange(self, name, api_key, api_secret):
log.info('setup_exchange', name=name)
if name not in SUPPORTED_EXCHANGES:
return False, 'Attempted to register unsupported exchange {}'.format(
name)
if getattr(self, name) is not None:
return False, 'Exchange {} is already registered'.format(name)
secret_data = {}
secret_data[name] = {
'api_key': api_key,
'api_secret': api_secret,
}
self.initialize_exchanges(secret_data)
exchange = getattr(self, name)
result, message = exchange.validate_api_key()
if not result:
log.error(
'Failed to validate API key for exchange',
name=name,
error=message,
)
self.delete_exchange_data(name)
return False, message
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return True, ''
def delete_exchange_data(self, name):
self.connected_exchanges.remove(name)
self.trades_historian.set_exchange(name, None)
delattr(self, name)
setattr(self, name, None)
def remove_exchange(self, name):
if getattr(self, name) is None:
return False, 'Exchange {} is not registered'.format(name)
self.delete_exchange_data(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def shutdown(self):
log.info("Shutting Down")
self.shutdown_event.set()
def test_data_init_and_password(data_dir, username):
"""DB Creation logic and tables at start testing"""
msg_aggregator = MessagesAggregator()
# Creating a new data dir should work
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
assert os.path.exists(os.path.join(data_dir, username))
# Trying to re-create it should throw
with pytest.raises(AuthenticationError):
data.unlock(username, '123', create_new=True)
# Trying to unlock a non-existing user without create_new should throw
with pytest.raises(AuthenticationError):
data.unlock('otheruser', '123', create_new=False)
# now relogin and check all tables are there
del data
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=False)
cursor = data.db.conn.cursor()
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
results = cursor.fetchall()
results = [result[0] for result in results]
assert set(results) == set(TABLES_AT_INIT)
# finally logging in with wrong password should also fail
del data
data = DataHandler(data_dir, msg_aggregator)
with pytest.raises(AuthenticationError):
data.unlock(username, '1234', create_new=False)
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise AssertionError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def reset_after_failed_account_creation_or_login(self) -> None:
"""If the account creation or login failed make sure that the Rotki instance is clear
Tricky instances are when after either failed premium credentials or user refusal
to sync premium databases we relogged in.
"""
self.cryptocompare.db = None
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
given_ethereum_modules: Optional[List[str]] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
if given_ethereum_modules is None:
ethereum_modules = ['makerdao'] # Default: ALL
else:
ethereum_modules = given_ethereum_modules
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
maybe_submit_usage_analytics(settings.submit_usage_analytics)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
alethio = Alethio(
database=self.data.db,
msg_aggregator=self.msg_aggregator,
all_eth_tokens=self.all_eth_tokens,
)
historical_data_start = settings.historical_data_start
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=self.cryptocompare,
)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethereum_manager=ethereum_manager,
msg_aggregator=self.msg_aggregator,
alethio=alethio,
greenlet_manager=self.greenlet_manager,
eth_modules=ethereum_modules,
)
self.ethereum_analyzer = EthereumAnalyzer(
ethereum_manager=ethereum_manager,
database=self.data.db,
)
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.user_is_logged_in = True
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
del self.chain_manager
self.exchange_manager.delete_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
del self.data_importer
if self.premium is not None:
del self.premium
self.data.logout()
self.password = ''
self.cryptocompare.unset_database()
# Make sure no messages leak to other user sessions
self.msg_aggregator.consume_errors()
self.msg_aggregator.consume_warnings()
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, credentials: PremiumCredentials) -> None:
"""
Sets the premium credentials for Rotki
Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(credentials)
else:
self.premium = premium_create_and_verify(credentials)
self.data.db.set_rotkehlchen_premium(credentials)
def start(self) -> gevent.Greenlet:
return gevent.spawn(self.main_loop)
def main_loop(self) -> None:
"""Rotki main loop that fires often and manages many different tasks
Each task remembers the last time it run sucesfully and know how often it
should run. So each task manages itself.
"""
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
if self.user_is_logged_in:
log.debug('Main loop start')
self.premium_sync_manager.maybe_upload_data_to_server()
self.ethereum_analyzer.analyze_ethereum_transactions()
log.debug('Main loop end')
def add_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> BlockchainBalancesUpdate:
"""Adds new blockchain accounts
Adds the accounts to the blockchain instance and queries them to get the
updated balances. Also adds them in the DB
May raise:
- EthSyncError from modify_blockchain_account
- InputError if the given accounts list is empty.
- TagConstraintError if any of the given account data contain unknown tags.
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
"""
self.data.db.ensure_tags_exist(
given_data=account_data,
action='adding',
data_type='blockchain accounts',
)
address_type = blockchain.get_address_type()
updated_balances = self.chain_manager.add_blockchain_accounts(
blockchain=blockchain,
accounts=[address_type(entry.address) for entry in account_data],
)
self.data.db.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return updated_balances
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError(
'Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(
self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}', )
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return None
def remove_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
) -> BlockchainBalancesUpdate:
"""Removes blockchain accounts
Removes the accounts from the blockchain instance and queries them to get
the updated balances. Also removes them from the DB
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- InputError if a non-existing account was given to remove
"""
balances_update = self.chain_manager.remove_blockchain_accounts(
blockchain=blockchain,
accounts=accounts,
)
self.data.db.remove_blockchain_accounts(blockchain, accounts)
return balances_update
def add_owned_eth_tokens(
self,
tokens: List[EthereumToken],
) -> BlockchainBalancesUpdate:
"""Adds tokens to the blockchain state and updates balance of all accounts
May raise:
- InputError if some of the tokens already exist
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- EthSyncError if querying the token balances through a provided ethereum
client and the chain is not synced
"""
new_data = self.chain_manager.track_new_tokens(tokens)
self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens)
return new_data
def remove_owned_eth_tokens(
self,
tokens: List[EthereumToken],
) -> BlockchainBalancesUpdate:
"""
Removes tokens from the state and stops their balance from being tracked
for each account
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- EthSyncError if querying the token balances through a provided ethereum
client and the chain is not synced
"""
new_data = self.chain_manager.remove_eth_tokens(tokens)
self.data.write_owned_eth_tokens(self.chain_manager.owned_eth_tokens)
return new_data
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
) = self.trades_historian.get_history(
start_ts=start_ts,
end_ts=end_ts,
has_premium=self.premium is not None,
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result, error_or_empty
def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]:
result = {}
balances = self.data.get_fiat_balances()
for currency, str_amount in balances.items():
amount = FVal(str_amount)
usd_rate = Inquirer().query_fiat_pair(currency, A_USD)
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate,
}
return result
def query_balances(
self,
requested_save_data: bool = True,
timestamp: Timestamp = None,
ignore_cache: bool = False,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are saved in the DB.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
If ignore_cache is True then all underlying calls that have a cache ignore it
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange_balances, _ = exchange.query_balances(
ignore_cache=ignore_cache)
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange.name] = exchange_balances
try:
blockchain_result = self.chain_manager.query_balances(
blockchain=None,
ignore_cache=ignore_cache,
)
balances['blockchain'] = {
asset: balance.to_dict()
for asset, balance in blockchain_result.totals.items()
}
except (RemoteError, EthSyncError) as e:
problem_free = False
log.error(f'Querying blockchain balances failed due to: {str(e)}')
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
balances = account_for_manually_tracked_balances(db=self.data.db,
balances=balances)
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
stats: Dict[str, Any] = {
'location': {},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data and self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.accountant.events.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
with self.lock:
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(
settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.kraken_account_type is not None:
kraken = self.exchange_manager.get('kraken')
if kraken:
kraken.set_account_type(
settings.kraken_account_type) # type: ignore
self.data.db.set_settings(settings)
return True, ''
def get_settings(self) -> DBSettings:
"""Returns the db settings with a check whether premium is active or not"""
db_settings = self.data.db.get_settings(
have_premium=self.premium is not None)
return db_settings
def setup_exchange(
self,
name: str,
api_key: ApiKey,
api_secret: ApiSecret,
passphrase: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret and optionally a passphrase
By default the api keys are always validated unless validate is False.
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
passphrase=passphrase,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(name,
api_key,
api_secret,
passphrase=passphrase)
return is_success, msg
def remove_exchange(self, name: str) -> Tuple[bool, str]:
if not self.exchange_manager.has_exchange(name):
return False, 'Exchange {} is not registered'.format(name)
self.exchange_manager.delete_exchange(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
self.data.db.delete_used_query_range_for_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result[
'eth_node_connection'] = self.chain_manager.ethereum.connected
result[
'history_process_start_ts'] = self.accountant.started_processing_timestamp
result[
'history_process_current_ts'] = self.accountant.currently_processing_timestamp
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
def test_writing_fetching_data(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
data.db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address='1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')],
)
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D'),
BlockchainAccountData(
address='0x80B369799104a47e98A553f3329812a44A7FaCDc'),
],
)
accounts = data.db.get_blockchain_accounts()
assert isinstance(accounts, BlockchainAccounts)
assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
# See that after addition the address has been checksummed
assert set(accounts.eth) == {
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
'0x80B369799104a47e98A553f3329812a44A7FaCDc',
}
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member
data.db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[
BlockchainAccountData(
address='0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
],
)
# Remove non-existing account
with pytest.raises(InputError):
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0x136029d76af6fE4A356528e4Dc66B2C18123597D'],
)
# Remove existing account
data.db.remove_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
['0xd36029d76af6fE4A356528e4Dc66B2C18123597D'],
)
accounts = data.db.get_blockchain_accounts()
assert accounts.eth == ['0x80B369799104a47e98A553f3329812a44A7FaCDc']
result, _ = data.add_ignored_assets([A_DAO])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert result
result, _ = data.add_ignored_assets([A_DOGE])
assert not result
ignored_assets = data.db.get_ignored_assets()
assert all(isinstance(asset, Asset) for asset in ignored_assets)
assert set(ignored_assets) == {A_DAO, A_DOGE}
# Test removing asset that is not in the list
result, msg = data.remove_ignored_assets([A_RDN])
assert 'not in ignored assets' in msg
assert not result
result, _ = data.remove_ignored_assets([A_DOGE])
assert result
assert data.db.get_ignored_assets() == [A_DAO]
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result.last_write_ts
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
expected_dict = {
'have_premium': False,
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_endpoint': 'http://localhost:8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO,
'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
'main_currency': DEFAULT_MAIN_CURRENCY.identifier,
'anonymized_logs': DEFAULT_ANONYMIZED_LOGS,
'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT,
'last_data_upload_ts': 0,
'premium_should_sync': False,
'submit_usage_analytics': True,
'last_write_ts': 0,
'kraken_account_type': DEFAULT_KRAKEN_ACCOUNT_TYPE,
'active_modules': DEFAULT_ACTIVE_MODULES,
'frontend_settings': '',
}
assert len(expected_dict) == len(
DBSettings()), 'One or more settings are missing'
# Make sure that results are the same. Comparing like this since we ignore last
# write ts check
result_dict = result._asdict()
for key, value in expected_dict.items():
assert key in result_dict
if key != 'last_write_ts':
assert value == result_dict[key]
def test_add_margin_positions(data_dir, username):
"""Test that adding and retrieving margin positions from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
margin1 = MarginPosition(
location=Location.BITMEX,
open_time=1451606400,
close_time=1451616500,
profit_loss=FVal('1.0'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin2 = MarginPosition(
location=Location.BITMEX,
open_time=1451626500,
close_time=1451636500,
profit_loss=FVal('0.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
margin3 = MarginPosition(
location=Location.POLONIEX,
open_time=1452636501,
close_time=1459836501,
profit_loss=FVal('2.5'),
pl_currency=A_BTC,
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_margin_positions([margin1, margin2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2]
# Add the last 2 margins. Since margin2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_margin_positions([margin2, margin3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_margins = data.db.get_margin_positions()
assert returned_margins == [margin1, margin2, margin3]
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_BTC,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SDC,
quote_asset=A_SDT2,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
), Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
base_asset=A_SUSHI,
quote_asset=A_1INCH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {A_USD, A_ETH, A_DAI, A_BTC, A_XMR, A_SDC, A_SDT2, A_SUSHI, A_1INCH} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 0
def test_add_asset_movements(data_dir, username, caplog):
"""Test that adding and retrieving asset movements from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
movement1 = AssetMovement(
location=Location.BITMEX,
category=AssetMovementCategory.DEPOSIT,
address=None,
transaction_id=None,
timestamp=1451606400,
asset=A_BTC,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0')),
link='',
)
movement2 = AssetMovement(
location=Location.POLONIEX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xfoo',
transaction_id='0xboo',
timestamp=1451608501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
movement3 = AssetMovement(
location=Location.BITTREX,
category=AssetMovementCategory.WITHDRAWAL,
address='0xcoo',
transaction_id='0xdoo',
timestamp=1461708501,
asset=A_ETH,
amount=FVal('1.0'),
fee_asset=A_EUR,
fee=Fee(FVal('0.01')),
link='',
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_asset_movements([movement1, movement2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2]
# Add the last 2 movements. Since movement2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_asset_movements([movement2, movement3])
assert (
'Did not add "withdrawal of ETH with id 94405f38c7b86dd2e7943164d'
'67ff44a32d56cef25840b3f5568e23c037fae0a'
) in caplog.text
returned_movements = data.db.get_asset_movements()
assert returned_movements == [movement1, movement2, movement3]
def setup_db_for_xpub_tests(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
data.db.add_tag('public', 'foooo', 'ffffff', '000000')
data.db.add_tag('desktop', 'boooo', 'ffffff', '000000')
xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501
derivation_path = 'm/0/0/0'
xpub_data1 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
label='xpub1',
tags=['public', 'desktop'],
)
data.db.ensure_tags_exist([xpub_data1],
action='adding',
data_type='bitcoin_xpub')
insert_tag_mappings( # if we got tags add them to the xpub
cursor=data.db.conn.cursor(),
data=[xpub_data1],
object_reference_keys=['xpub.xpub', 'derivation_path'],
)
data.db.add_bitcoin_xpub(xpub_data1)
addr1 = '1LZypJUwJJRdfdndwvDmtAjrVYaHko136r'
addr2 = '1MKSdDCtBSXiE49vik8xUG2pTgTGGh5pqe'
addr3 = '12wxFzpjdymPk3xnHmdDLCTXUT9keY3XRd'
addr4 = '16zNpyv8KxChtjXnE5nYcPqcXcrSQXX2JW'
all_addresses = [addr1, addr2, addr3, addr4]
account_data = [
BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]
]
data.db.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
account_data=account_data,
)
insert_tag_mappings( # if we got tags add them to the existing addresses too
cursor=data.db.conn.cursor(),
data=account_data,
object_reference_keys=['address'],
)
data.db.ensure_xpub_mappings_exist(
xpub=xpub,
derivation_path=derivation_path,
derived_addresses_data=[
XpubDerivedAddressData(0, 0, addr1, ZERO),
XpubDerivedAddressData(0, 1, addr2, ZERO),
],
)
xpub = 'zpub6quTRdxqWmerHdiWVKZdLMp9FY641F1F171gfT2RS4D1FyHnutwFSMiab58Nbsdu4fXBaFwpy5xyGnKZ8d6xn2j4r4yNmQ3Yp3yDDxQUo3q' # noqa: E501
derivation_path = 'm/0'
xpub_data2 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
)
data.db.add_bitcoin_xpub(xpub_data2)
addr1 = 'bc1qc3qcxs025ka9l6qn0q5cyvmnpwrqw2z49qwrx5'
addr2 = 'bc1qnus7355ecckmeyrmvv56mlm42lxvwa4wuq5aev'
addr3 = 'bc1qup7f8g5k3h5uqzfjed03ztgn8hhe542w69wc0g'
addr4 = 'bc1qr4r8vryfzexvhjrx5fh5uj0s2ead8awpqspqra'
all_addresses.extend([addr1, addr2, addr3, addr4])
data.db.add_blockchain_accounts(
blockchain=SupportedBlockchain.BITCOIN,
account_data=[
BlockchainAccountData(x) for x in [addr1, addr2, addr3, addr4]
],
)
data.db.ensure_xpub_mappings_exist(
xpub=xpub,
derivation_path=derivation_path,
derived_addresses_data=[
XpubDerivedAddressData(1, 0, addr1, ZERO),
XpubDerivedAddressData(1, 1, addr2, ZERO),
XpubDerivedAddressData(1, 2, addr3, ZERO),
XpubDerivedAddressData(1, 3, addr4, ZERO),
],
)
# Finally also add the same xpub as xpub1 with no derivation path
xpub = 'xpub68V4ZQQ62mea7ZUKn2urQu47Bdn2Wr7SxrBxBDDwE3kjytj361YBGSKDT4WoBrE5htrSB8eAMe59NPnKrcAbiv2veN5GQUmfdjRddD1Hxrk' # noqa: E501
derivation_path = None
xpub_data3 = XpubData(
xpub=HDKey.from_xpub(xpub=xpub, path='m'),
derivation_path=derivation_path,
)
data.db.add_bitcoin_xpub(xpub_data3)
return data.db, xpub_data1, xpub_data2, xpub_data3, all_addresses
def test_query_trades_including_ammswaps(data_dir, username):
"""Test that querying trades succesfully queries from both the trades and ammswaps"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trades = [
Trade(
timestamp=1,
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_USDC,
trade_type=TradeType.BUY,
amount=FVal(1),
rate=Price(FVal(1.5)),
fee=None,
fee_currency=None,
link='',
notes=None,
),
Trade(
timestamp=2,
location=Location.KRAKEN,
base_asset=A_BTC,
quote_asset=A_EUR,
trade_type=TradeType.SELL,
amount=FVal(1),
rate=Price(FVal(1.5)),
fee=None,
fee_currency=None,
link='',
notes=None,
),
Trade(
timestamp=3,
location=Location.POLONIEX,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=FVal(2),
rate=Price(FVal(1.5)),
fee=None,
fee_currency=None,
link='',
notes=None,
),
]
swaps = [
AMMSwap(
tx_hash='0x1',
log_index=1,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=2,
location=Location.UNISWAP,
token0=A_DAI,
token1=A_USDC,
amount0_in=FVal(1),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(2),
),
AMMSwap(
tx_hash='0x2',
log_index=2,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=3,
location=Location.BALANCER,
token0=A_UNI,
token1=A_USDC,
amount0_in=FVal(1),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(1.5),
),
AMMSwap(
tx_hash='0x2',
log_index=3,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=3,
location=Location.BALANCER,
token0=A_USDC,
token1=A_DAI,
amount0_in=FVal(1.5),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(3.5),
),
AMMSwap(
tx_hash='0x3',
log_index=1,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=4,
location=Location.SUSHISWAP,
token0=A_UNI,
token1=A_DAI,
amount0_in=ZERO,
amount1_in=FVal(2),
amount0_out=FVal(5),
amount1_out=ZERO,
),
AMMSwap(
tx_hash='0x3',
log_index=2,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=4,
location=Location.SUSHISWAP,
token0=A_DAI,
token1=A_USDC,
amount0_in=FVal(5),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(4.95),
),
AMMSwap(
tx_hash='0x3',
log_index=3,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=4,
location=Location.SUSHISWAP,
token0=A_GNO,
token1=A_USDC,
amount0_in=ZERO,
amount1_in=FVal(4.95),
amount0_out=FVal(8.2),
amount1_out=ZERO,
),
AMMSwap(
tx_hash='0x4',
log_index=5,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=11,
location=Location.UNISWAP,
token0=A_GNO,
token1=A_USDC,
amount0_in=FVal(5),
amount1_in=FVal(6),
amount0_out=ZERO,
amount1_out=FVal(4.95),
),
AMMSwap(
tx_hash='0x4',
log_index=10,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=11,
location=Location.UNISWAP,
token0=A_USDC,
token1=A_UNI,
amount0_in=FVal(4.95),
amount1_in=ZERO,
amount0_out=ZERO,
amount1_out=FVal(5.4),
),
AMMSwap(
tx_hash='0x5',
log_index=1,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=14,
location=Location.SUSHISWAP,
token0=A_UNI,
token1=A_USDC,
amount0_in=FVal(4.5),
amount1_in=FVal(6),
amount0_out=ZERO,
amount1_out=FVal(3.2),
),
AMMSwap(
tx_hash='0x5',
log_index=3,
address='0xfoo',
from_address='0xfrom',
to_address='0xto',
timestamp=14,
location=Location.SUSHISWAP,
token0=A_USDC,
token1=A_GNO,
amount0_in=FVal(5.55),
amount1_in=ZERO,
amount0_out=FVal(2.15),
amount1_out=FVal(5.4),
),
]
data.db.add_trades(trades)
data.db.add_amm_swaps(swaps)
swap1_trade = Trade(
timestamp=swaps[0].timestamp,
location=swaps[0].location,
base_asset=swaps[0].token1,
quote_asset=swaps[0].token0,
trade_type=TradeType.BUY,
amount=swaps[0].amount1_out,
rate=swaps[0].amount0_in / swaps[0].amount1_out,
fee=None,
fee_currency=None,
link=swaps[0].tx_hash,
notes=None,
)
swap2_trade = Trade(
timestamp=swaps[1].timestamp,
location=swaps[1].location,
base_asset=swaps[2].token1,
quote_asset=swaps[1].token0,
trade_type=TradeType.BUY,
amount=swaps[2].amount1_out,
rate=swaps[1].amount0_in / swaps[2].amount1_out,
fee=None,
fee_currency=None,
link=swaps[1].tx_hash,
notes=None,
)
swap3_trade = Trade(
timestamp=swaps[3].timestamp,
location=swaps[3].location,
base_asset=swaps[5].token0,
quote_asset=swaps[3].token1,
trade_type=TradeType.BUY,
amount=swaps[5].amount0_out,
rate=swaps[3].amount1_in / swaps[5].amount0_out,
fee=None,
fee_currency=None,
link=swaps[3].tx_hash,
notes=None,
)
swap4_trade1 = Trade(
timestamp=swaps[6].timestamp,
location=swaps[6].location,
base_asset=swaps[7].token1,
quote_asset=swaps[6].token0,
trade_type=TradeType.BUY,
amount=swaps[7].amount1_out / 2,
rate=swaps[6].amount0_in / (swaps[7].amount1_out / 2),
fee=None,
fee_currency=None,
link=swaps[6].tx_hash,
notes=None,
)
swap4_trade2 = Trade(
timestamp=swaps[6].timestamp,
location=swaps[6].location,
base_asset=swaps[7].token1,
quote_asset=swaps[6].token1,
trade_type=TradeType.BUY,
amount=swaps[7].amount1_out / 2,
rate=swaps[6].amount1_in / (swaps[7].amount1_out / 2),
fee=None,
fee_currency=None,
link=swaps[6].tx_hash,
notes=None,
)
swap5_trade1 = Trade(
timestamp=swaps[8].timestamp,
location=swaps[8].location,
base_asset=swaps[9].token1,
quote_asset=swaps[8].token1,
trade_type=TradeType.BUY,
amount=swaps[9].amount1_out,
rate=swaps[8].amount1_in / swaps[9].amount1_out,
fee=None,
fee_currency=None,
link=swaps[8].tx_hash,
notes=None,
)
swap5_trade2 = Trade(
timestamp=swaps[8].timestamp,
location=swaps[8].location,
base_asset=swaps[9].token0,
quote_asset=swaps[8].token0,
trade_type=TradeType.BUY,
amount=swaps[9].amount0_out,
rate=swaps[8].amount0_in / swaps[9].amount0_out,
fee=None,
fee_currency=None,
link=swaps[8].tx_hash,
notes=None,
)
# Get all trades
returned_trades = data.db.get_trades(filter_query=TradesFilterQuery.make(),
has_premium=True)
assert len(returned_trades) == 10
assert returned_trades[0] == trades[0]
assert returned_trades[2] == trades[1]
assert returned_trades[4] == trades[2]
assert_trades_equal(returned_trades[1], swap1_trade)
assert_trades_equal(returned_trades[3], swap2_trade)
assert_trades_equal(returned_trades[5], swap3_trade)
assert_trades_equal(returned_trades[6], swap4_trade2)
assert_trades_equal(returned_trades[7], swap4_trade1)
assert_trades_equal(returned_trades[8], swap5_trade1)
assert_trades_equal(returned_trades[9], swap5_trade2)
# Get last 5 trades
returned_trades = data.db.get_trades(
filter_query=TradesFilterQuery.make(limit=5, offset=5),
has_premium=True,
)
assert len(returned_trades) == 5
assert_trades_equal(returned_trades[0], swap3_trade)
assert_trades_equal(returned_trades[1], swap4_trade2)
assert_trades_equal(returned_trades[2], swap4_trade1)
assert_trades_equal(returned_trades[3], swap5_trade1)
assert_trades_equal(returned_trades[4], swap5_trade2)
# Get first 5 trades that are in uniswap and that buy USDC
returned_trades = data.db.get_trades(
filter_query=TradesFilterQuery.make(
limit=5,
offset=0,
location=Location.UNISWAP,
base_asset=A_USDC,
),
has_premium=True,
)
assert len(returned_trades) == 1
assert_trades_equal(returned_trades[0], swap1_trade)
# Get all trades with quote asset USDC
returned_trades = data.db.get_trades(
filter_query=TradesFilterQuery.make(quote_asset=A_USDC),
has_premium=True,
)
assert len(returned_trades) == 3
assert_trades_equal(returned_trades[0], trades[0])
assert_trades_equal(returned_trades[1], swap4_trade2)
assert_trades_equal(returned_trades[2], swap5_trade1)
# Get all trades as non premium user with 2 free trades as limit
limit_patch = patch(
target='rotkehlchen.db.dbhandler.FREE_TRADES_LIMIT',
new=2,
)
with limit_patch:
returned_trades, total_found = data.db.get_trades_and_limit_info(
filter_query=TradesFilterQuery.make(),
has_premium=False,
)
# trades should be the latest 2
assert total_found == 3 # the 3 normal trades -- free users don't see swaps
assert len(returned_trades) == 2
assert_trades_equal(returned_trades[0], trades[1])
assert_trades_equal(returned_trades[1], trades[2])
# Get filtered trades as non premium user with 2 free trades as limit
limit_patch = patch(
target='rotkehlchen.db.dbhandler.FREE_TRADES_LIMIT',
new=2,
)
with limit_patch:
returned_trades, total_found = data.db.get_trades_and_limit_info(
filter_query=TradesFilterQuery.make(from_ts=1, to_ts=2),
has_premium=False,
)
# trades should be the second one since the free limit includes the last 2 only
assert total_found == 2, 'total found for filter should be 2'
assert len(returned_trades) == 1
assert_trades_equal(returned_trades[0], trades[1])
def test_writting_fetching_external_trades(data_dir, username):
data = DataHandler(data_dir)
data.unlock(username, '123', create_new=True)
# add 2 trades and check they are in the DB
trade1 = {
'otc_timestamp': '10/03/2018 23:30',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '10',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link',
'otc_notes': 'a note',
}
trade2 = {
'otc_timestamp': '10/03/2018 23:35',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '5',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link 2',
'otc_notes': 'a note 2',
}
result, _, = data.add_external_trade(trade1)
assert result
result, _ = data.add_external_trade(trade2)
assert result
result = data.get_external_trades()
del result[0]['id']
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# edit a trade and check the edit made it in the DB
trade1['otc_rate'] = '120'
trade1['otc_id'] = 1
result, _ = data.edit_external_trade(trade1)
assert result
result = data.get_external_trades()
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# try to edit a non-existing trade
trade1['otc_rate'] = '160'
trade1['otc_id'] = 5
result, _ = data.edit_external_trade(trade1)
assert not result
trade1['otc_rate'] = '120'
trade1['otc_id'] = 1
result = data.get_external_trades()
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# try to delete non-existing trade
result, _ = data.delete_external_trade(6)
assert not result
# delete an external trade
result, _ = data.delete_external_trade(1)
result = data.get_external_trades()
del result[0]['id']
assert result[0] == from_otc_trade(trade2)
def test_writting_fetching_data(data_dir, username):
data = DataHandler(data_dir)
data.unlock(username, '123', create_new=True)
tokens = ['GNO', 'RDN']
data.write_owned_eth_tokens(tokens)
result = data.db.get_owned_tokens()
assert set(tokens) == set(result)
data.add_blockchain_account('BTC', '1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')
data.add_blockchain_account('ETH',
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
data.add_blockchain_account('ETH',
'0x80b369799104a47e98a553f3329812a44a7facdc')
accounts = data.db.get_blockchain_accounts()
assert len(accounts) == 2
assert accounts['BTC'] == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
assert set(accounts['ETH']) == set([
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
])
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError):
data.add_blockchain_account(
'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
# Remove non-existing account
with pytest.raises(InputError):
data.remove_blockchain_account(
'ETH', '0x136029d76af6fE4A356528e4Dc66B2C18123597D')
# Remove existing account
data.remove_blockchain_account(
'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
accounts = data.db.get_blockchain_accounts()
assert accounts['ETH'] == [
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
]
result, _ = data.add_ignored_asset('DAO')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert not result
assert set(data.db.get_ignored_assets()) == set(['DAO', 'DOGE'])
result, _ = data.remove_ignored_asset('XXX')
assert not result
result, _ = data.remove_ignored_asset('DOGE')
assert result
assert data.db.get_ignored_assets() == ['DAO']
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result['last_write_ts']
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
del result['last_write_ts']
assert result == {
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_port': '8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'db_version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': True,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
}
# Check setting non-existing settings. Should be ignored
_, msg = data.set_settings({'nonexisting_setting': 1}, accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
_, msg = data.set_settings(
{
'nonexisting_setting': 1,
'eth_rpc_port': '8555',
'ui_floating_precision': 3,
},
accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
# Now check nothing funny made it in the db
result = data.db.get_settings()
assert result['eth_rpc_port'] == '8555'
assert result['ui_floating_precision'] == 3
assert 'nonexisting_setting' not in result
def test_add_and_get_yearn_vault_events(data_dir, username):
"""Test that get yearn vault events works fine and returns only events for what we need"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
addr1 = make_ethereum_address()
addr1_events = [YearnVaultEvent(
event_type='deposit',
from_asset=A_DAI,
from_value=Balance(amount=FVal(1), usd_value=FVal(1)),
to_asset=Asset('yDAI'),
to_value=Balance(amount=FVal(1), usd_value=FVal(1)),
realized_pnl=None,
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436',
log_index=1,
), YearnVaultEvent(
event_type='withdraw',
from_asset=Asset('yDAI'),
from_value=Balance(amount=FVal(1), usd_value=FVal(1)),
to_asset=A_DAI,
to_value=Balance(amount=FVal(1), usd_value=FVal(1)),
realized_pnl=Balance(amount=FVal('0.01'), usd_value=FVal('0.01')),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce',
log_index=2,
)]
data.db.add_yearn_vaults_events(address=addr1, events=addr1_events)
addr2 = make_ethereum_address()
addr2_events = [YearnVaultEvent(
event_type='deposit',
from_asset=A_DAI,
from_value=Balance(amount=FVal(1), usd_value=FVal(1)),
to_asset=Asset('yDAI'),
to_value=Balance(amount=FVal(1), usd_value=FVal(1)),
realized_pnl=None,
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55',
log_index=1,
), YearnVaultEvent(
event_type='withdraw',
from_asset=Asset('yDAI'),
from_value=Balance(amount=FVal(1), usd_value=FVal(1)),
to_asset=A_DAI,
to_value=Balance(amount=FVal(1), usd_value=FVal(1)),
realized_pnl=Balance(amount=FVal('0.01'), usd_value=FVal('0.01')),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=2,
)]
data.db.add_yearn_vaults_events(address=addr2, events=addr2_events)
events = data.db.get_yearn_vaults_events(address=addr1, vault=YEARN_VAULTS['yDAI'])
assert events == addr1_events
events = data.db.get_yearn_vaults_events(address=addr2, vault=YEARN_VAULTS['yDAI'])
assert events == addr2_events
def test_query_owned_assets(data_dir, username):
"""Test the get_owned_assets with also an unknown asset in the DB"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
balances = deepcopy(asset_balances)
balances.extend([
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1488326400),
asset=A_BTC,
amount='1',
usd_value='1222.66',
),
DBAssetBalance(
category=BalanceType.ASSET,
time=Timestamp(1489326500),
asset=A_XMR,
amount='2',
usd_value='33.8',
),
])
data.db.add_multiple_balances(balances)
cursor = data.db.conn.cursor()
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value, category) '
' VALUES(?, ?, ?, ?, ?)',
(1469326500, 'ADSADX', '10.1', '100.5', 'A'),
)
data.db.conn.commit()
# also make sure that assets from trades are included
data.db.add_trades([
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(99),
location=Location.EXTERNAL,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SDC_SDT-2'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(3),
location=Location.EXTERNAL,
pair=TradePair('SUSHI_1INCH'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(2)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
Trade(
timestamp=Timestamp(1),
location=Location.EXTERNAL,
pair=TradePair('UNKNOWNTOKEN_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_BTC,
link='',
notes='',
),
])
assets_list = data.db.query_owned_assets()
assert set(assets_list) == {
A_USD, A_ETH, A_DAI, A_BTC, A_XMR,
Asset('SDC'),
Asset('SDT-2'),
Asset('SUSHI'),
Asset('1INCH')
} # noqa: E501
assert all(isinstance(x, Asset) for x in assets_list)
warnings = data.db.msg_aggregator.consume_warnings()
assert len(warnings) == 1
assert 'Unknown/unsupported asset ADSADX' in warnings[0]
def test_add_and_get_aave_events(data_dir, username):
"""Test that get aave events works fine and returns only events for what we need"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
addr1 = make_ethereum_address()
addr1_events = [AaveDepositWithdrawalEvent(
event_type='deposit',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x01653e88600a6492ad6e9ae2af415c990e623479057e4e93b163e65cfb2d4436',
log_index=1,
), AaveDepositWithdrawalEvent(
event_type='withdrawal',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x4147da3e5d3c0565a99192ce0b32182ab30b8e1067921d9b2a8ef3bd60b7e2ce',
log_index=2,
)]
data.db.add_aave_events(address=addr1, events=addr1_events)
addr2 = make_ethereum_address()
addr2_events = [AaveDepositWithdrawalEvent(
event_type='deposit',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x8c094d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55',
log_index=1,
), AaveDepositWithdrawalEvent(
event_type='withdrawal',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x58c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=2,
)]
data.db.add_aave_events(address=addr2, events=addr2_events)
# addr3 has all types of aave events so we test serialization/deserialization
addr3 = make_ethereum_address()
addr3_events = [AaveDepositWithdrawalEvent(
event_type='deposit',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=1,
timestamp=Timestamp(1),
tx_hash='0x9e394d58f33e8dedcd348cb33b58f3bd447602f1fecb99e51b1c2868029eab55',
log_index=1,
), AaveDepositWithdrawalEvent(
event_type='withdrawal',
asset=A_DAI,
atoken=A_ADAI_V1,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=2,
timestamp=Timestamp(2),
tx_hash='0x4c167445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=2,
), AaveInterestEvent(
event_type='interest',
asset=A_WBTC,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=4,
timestamp=Timestamp(4),
tx_hash='0x49c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=4,
), AaveBorrowEvent(
event_type='borrow',
asset=A_ETH,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=5,
timestamp=Timestamp(5),
tx_hash='0x19c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=5,
borrow_rate_mode='stable',
borrow_rate=FVal('0.05233232323423432'),
accrued_borrow_interest=FVal('5.112234'),
), AaveRepayEvent(
event_type='repay',
asset=A_MANA,
value=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=6,
timestamp=Timestamp(6),
tx_hash='0x29c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
log_index=6,
fee=Balance(amount=FVal('0.1'), usd_value=FVal('0.1')),
), AaveLiquidationEvent(
event_type='liquidation',
collateral_asset=A_ETH,
collateral_balance=Balance(amount=FVal(1), usd_value=FVal(1)),
principal_asset=A_ETH,
principal_balance=Balance(amount=FVal(1), usd_value=FVal(1)),
block_number=7,
log_index=7,
timestamp=Timestamp(7),
tx_hash='0x39c67445d26679623f9b7d56a8be260a275cb6744a1c1ae5a8d6883a5a5c03de',
)]
data.db.add_aave_events(address=addr3, events=addr3_events)
events = data.db.get_aave_events(address=addr1, atoken=A_ADAI_V1)
assert events == addr1_events
events = data.db.get_aave_events(address=addr2, atoken=A_ADAI_V1)
assert events == addr2_events
events = data.db.get_aave_events(address=addr3)
assert events == addr3_events
# check that all aave events are properly hashable (aka can go in a set)
test_set = set()
for event in addr3_events:
test_set.add(event)
assert len(test_set) == len(addr3_events)
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
# Initialize the AssetResolver singleton
AssetResolver(data_directory=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir,
coingecko=self.coingecko)
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='periodically_query_icons_until_all_cached',
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.lock.release()
self.shutdown_event = gevent.event.Event()
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
This runs during backend initialization so it should be as light as possible.
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
self.data_dir.mkdir(parents=True, exist_ok=True)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.main_loop_spawned = False
self.args = args
self.api_task_greenlets: List[gevent.Greenlet] = []
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.rotki_notifier = RotkiNotifier(
greenlet_manager=self.greenlet_manager)
self.msg_aggregator.rotki_notifier = self.rotki_notifier
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
# Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver
GlobalDBHandler(data_dir=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir,
coingecko=self.coingecko)
self.assets_updater = AssetsUpdater(self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
self.task_manager: Optional[TaskManager] = None
self.shutdown_event = gevent.event.Event()
def reset_after_failed_account_creation_or_login(self) -> None:
"""If the account creation or login failed make sure that the rotki instance is clear
Tricky instances are when after either failed premium credentials or user refusal
to sync premium databases we relogged in
"""
self.cryptocompare.db = None
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
sync_database: bool = True,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
sync_database=sync_database,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new,
initial_settings)
# Run the DB integrity check due to https://github.com/rotki/rotki/issues/3010
# TODO: Hopefully onece 3010 is handled this can go away
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='user DB data integrity check',
exception_is_error=False,
method=self.data.db.ensure_data_integrity,
)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
sync_database=sync_database,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_warning(
'Could not authenticate the rotki premium API keys found in the DB.'
' Has your subscription expired?', )
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
exception_is_error=False,
method=maybe_submit_usage_analytics,
data_dir=self.data_dir,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db,
msg_aggregator=self.msg_aggregator)
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
PriceHistorian().set_oracles_order(settings.historical_price_oracles)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
kusama_manager = SubstrateManager(
chain=SubstrateChain.KUSAMA,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_ksm_manager_on_startup(),
own_rpc_endpoint=settings.ksm_rpc_endpoint,
)
polkadot_manager = SubstrateManager(
chain=SubstrateChain.POLKADOT,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=POLKADOT_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_dot_manager_on_startup(),
own_rpc_endpoint=settings.dot_rpc_endpoint,
)
self.eth_transactions = EthTransactions(ethereum=ethereum_manager,
database=self.data.db)
self.covalent_avalanche = Covalent(
database=self.data.db,
msg_aggregator=self.msg_aggregator,
chain_id=chains_id['avalanche'],
)
avalanche_manager = AvalancheManager(
avaxrpc_endpoint="https://api.avax.network/ext/bc/C/rpc",
covalent=self.covalent_avalanche,
msg_aggregator=self.msg_aggregator,
)
Inquirer().inject_ethereum(ethereum_manager)
uniswap_v2_oracle = UniswapV2Oracle(ethereum_manager)
uniswap_v3_oracle = UniswapV3Oracle(ethereum_manager)
saddle_oracle = SaddleOracle(ethereum_manager)
Inquirer().add_defi_oracles(
uniswap_v2=uniswap_v2_oracle,
uniswap_v3=uniswap_v3_oracle,
saddle=saddle_oracle,
)
Inquirer().set_oracles_order(settings.current_price_oracles)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
kusama_manager=kusama_manager,
polkadot_manager=polkadot_manager,
avalanche_manager=avalanche_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
btc_derivation_gap_limit=settings.btc_derivation_gap_limit,
)
self.evm_tx_decoder = EVMTransactionDecoder(
database=self.data.db,
ethereum_manager=ethereum_manager,
eth_transactions=self.eth_transactions,
msg_aggregator=self.msg_aggregator,
)
self.evm_accounting_aggregator = EVMAccountingAggregator(
ethereum_manager=ethereum_manager,
msg_aggregator=self.msg_aggregator,
)
self.accountant = Accountant(
db=self.data.db,
msg_aggregator=self.msg_aggregator,
evm_accounting_aggregator=self.evm_accounting_aggregator,
premium=self.premium,
)
self.events_historian = EventsHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
evm_tx_decoder=self.evm_tx_decoder,
eth_transactions=self.eth_transactions,
)
self.task_manager = TaskManager(
max_tasks_num=DEFAULT_MAX_TASKS_NUM,
greenlet_manager=self.greenlet_manager,
api_task_greenlets=self.api_task_greenlets,
database=self.data.db,
cryptocompare=self.cryptocompare,
premium_sync_manager=self.premium_sync_manager,
chain_manager=self.chain_manager,
exchange_manager=self.exchange_manager,
eth_transactions=self.eth_transactions,
evm_tx_decoder=self.evm_tx_decoder,
deactivate_premium=self.deactivate_premium_status,
query_balances=self.query_balances,
)
DataMigrationManager(self).maybe_migrate_data()
self.greenlet_manager.spawn_and_track(
after_seconds=5,
task_name='periodically_query_icons_until_all_cached',
exception_is_error=False,
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def _logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
self.deactivate_premium_status()
self.greenlet_manager.clear()
del self.chain_manager
self.exchange_manager.delete_all_exchanges()
del self.accountant
del self.events_historian
del self.data_importer
self.data.logout()
self.password = ''
self.cryptocompare.unset_database()
# Make sure no messages leak to other user sessions
self.msg_aggregator.consume_errors()
self.msg_aggregator.consume_warnings()
self.task_manager = None
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def logout(self) -> None:
if self.task_manager is None: # no user logged in?
return
with self.task_manager.schedule_lock:
self._logout()
def set_premium_credentials(self, credentials: PremiumCredentials) -> None:
"""
Sets the premium credentials for rotki
Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(credentials)
else:
self.premium = premium_create_and_verify(credentials)
self.premium_sync_manager.premium = self.premium
self.accountant.activate_premium_status(self.premium)
self.chain_manager.activate_premium_status(self.premium)
self.data.db.set_rotkehlchen_premium(credentials)
def deactivate_premium_status(self) -> None:
"""Deactivate premium in the current session"""
self.premium = None
self.premium_sync_manager.premium = None
self.accountant.deactivate_premium_status()
self.chain_manager.deactivate_premium_status()
def delete_premium_credentials(self) -> Tuple[bool, str]:
"""Deletes the premium credentials for rotki"""
msg = ''
success = self.data.db.del_rotkehlchen_premium()
if success is False:
msg = 'The database was unable to delete the Premium keys for the logged-in user'
self.deactivate_premium_status()
return success, msg
def start(self) -> gevent.Greenlet:
assert not self.main_loop_spawned, 'Tried to spawn the main loop twice'
greenlet = gevent.spawn(self.main_loop)
self.main_loop_spawned = True
return greenlet
def main_loop(self) -> None:
"""rotki main loop that fires often and runs the task manager's scheduler"""
while self.shutdown_event.wait(
timeout=MAIN_LOOP_SECS_DELAY) is not True:
if self.task_manager is not None:
self.task_manager.schedule()
def get_blockchain_account_data(
self,
blockchain: SupportedBlockchain,
) -> Union[List[BlockchainAccountData], Dict[str, Any]]:
account_data = self.data.db.get_blockchain_account_data(blockchain)
if blockchain != SupportedBlockchain.BITCOIN:
return account_data
xpub_data = self.data.db.get_bitcoin_xpub_data()
addresses_to_account_data = {x.address: x for x in account_data}
address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping(
list(addresses_to_account_data.keys()), # type: ignore
)
xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {}
for address, xpub_entry in address_to_xpub_mappings.items():
if xpub_entry not in xpub_mappings:
xpub_mappings[xpub_entry] = []
xpub_mappings[xpub_entry].append(
addresses_to_account_data[address])
data: Dict[str, Any] = {'standalone': [], 'xpubs': []}
# Add xpub data
for xpub_entry in xpub_data:
data_entry = xpub_entry.serialize()
addresses = xpub_mappings.get(xpub_entry, None)
data_entry['addresses'] = addresses if addresses and len(
addresses) != 0 else None
data['xpubs'].append(data_entry)
# Add standalone addresses
for account in account_data:
if account.address not in address_to_xpub_mappings:
data['standalone'].append(account)
return data
def add_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> BlockchainBalancesUpdate:
"""Adds new blockchain accounts
Adds the accounts to the blockchain instance and queries them to get the
updated balances. Also adds them in the DB
May raise:
- EthSyncError from modify_blockchain_account
- InputError if the given accounts list is empty.
- TagConstraintError if any of the given account data contain unknown tags.
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
"""
self.data.db.ensure_tags_exist(
given_data=account_data,
action='adding',
data_type='blockchain accounts',
)
address_type = blockchain.get_address_type()
updated_balances = self.chain_manager.add_blockchain_accounts(
blockchain=blockchain,
accounts=[address_type(entry.address) for entry in account_data],
)
self.data.db.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return updated_balances
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError(
'Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(
self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}', )
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
def remove_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
) -> BlockchainBalancesUpdate:
"""Removes blockchain accounts
Removes the accounts from the blockchain instance and queries them to get
the updated balances. Also removes them from the DB
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- InputError if a non-existing account was given to remove
"""
balances_update = self.chain_manager.remove_blockchain_accounts(
blockchain=blockchain,
accounts=accounts,
)
eth_addresses: List[ChecksumEthAddress] = cast(
List[ChecksumEthAddress],
accounts) if blockchain == SupportedBlockchain.ETHEREUM else [
] # noqa: E501
with self.eth_transactions.wait_until_no_query_for(eth_addresses):
self.data.db.remove_blockchain_accounts(blockchain, accounts)
return balances_update
def get_history_query_status(self) -> Dict[str, str]:
if self.events_historian.progress < FVal('100'):
processing_state = self.events_historian.processing_state_name
progress = self.events_historian.progress / 2
elif self.accountant.first_processed_timestamp == -1:
processing_state = 'Processing all retrieved historical events'
progress = FVal(50)
else:
processing_state = 'Processing all retrieved historical events'
# start_ts is min of the query start or the first action timestamp since action
# processing can start well before query start to calculate cost basis
start_ts = min(
self.accountant.query_start_ts,
self.accountant.first_processed_timestamp,
)
diff = self.accountant.query_end_ts - start_ts
progress = 50 + 100 * (
FVal(self.accountant.currently_processing_timestamp - start_ts)
/ FVal(diff) / 2)
return {
'processing_state': str(processing_state),
'total_progress': str(progress)
}
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[int, str]:
error_or_empty, events = self.events_historian.get_history(
start_ts=start_ts,
end_ts=end_ts,
has_premium=self.premium is not None,
)
report_id = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
events=events,
)
return report_id, error_or_empty
def query_balances(
self,
requested_save_data: bool = False,
save_despite_errors: bool = False,
timestamp: Timestamp = None,
ignore_cache: bool = False,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are always saved in the DB,
if it is False then data are saved if self.data.should_save_balances()
is True.
If save_despite_errors is True then even if there is any error the snapshot
will be saved.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
If ignore_cache is True then all underlying calls that have a cache ignore it
Returns a dictionary with the queried balances.
"""
log.info(
'query_balances called',
requested_save_data=requested_save_data,
save_despite_errors=save_despite_errors,
)
balances: Dict[str, Dict[Asset, Balance]] = {}
problem_free = True
for exchange in self.exchange_manager.iterate_exchanges():
exchange_balances, error_msg = exchange.query_balances(
ignore_cache=ignore_cache)
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
self.msg_aggregator.add_message(
message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR,
data={
'location': exchange.name,
'error': error_msg
},
)
else:
location_str = str(exchange.location)
if location_str not in balances:
balances[location_str] = exchange_balances
else: # multiple exchange of same type. Combine balances
balances[location_str] = combine_dicts(
balances[location_str],
exchange_balances,
)
liabilities: Dict[Asset, Balance]
try:
blockchain_result = self.chain_manager.query_balances(
blockchain=None,
force_token_detection=ignore_cache,
ignore_cache=ignore_cache,
)
if len(blockchain_result.totals.assets) != 0:
balances[str(
Location.BLOCKCHAIN)] = blockchain_result.totals.assets
liabilities = blockchain_result.totals.liabilities
except (RemoteError, EthSyncError) as e:
problem_free = False
liabilities = {}
log.error(f'Querying blockchain balances failed due to: {str(e)}')
self.msg_aggregator.add_message(
message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR,
data={
'location': 'blockchain balances query',
'error': str(e)
},
)
manually_tracked_liabilities = get_manually_tracked_balances(
db=self.data.db,
balance_type=BalanceType.LIABILITY,
)
manual_liabilities_as_dict: DefaultDict[Asset,
Balance] = defaultdict(Balance)
for manual_liability in manually_tracked_liabilities:
manual_liabilities_as_dict[
manual_liability.asset] += manual_liability.value
liabilities = combine_dicts(liabilities, manual_liabilities_as_dict)
# retrieve loopring balances if module is activated
if self.chain_manager.get_module('loopring'):
try:
loopring_balances = self.chain_manager.get_loopring_balances()
except RemoteError as e:
problem_free = False
self.msg_aggregator.add_message(
message_type=WSMessageType.BALANCE_SNAPSHOT_ERROR,
data={
'location': 'loopring',
'error': str(e)
},
)
else:
if len(loopring_balances) != 0:
balances[str(Location.LOOPRING)] = loopring_balances
# retrieve nft balances if module is activated
nfts = self.chain_manager.get_module('nfts')
if nfts is not None:
try:
nft_mapping = nfts.get_balances(
addresses=self.chain_manager.queried_addresses_for_module(
'nfts'),
return_zero_values=False,
ignore_cache=False,
)
except RemoteError as e:
log.error(
f'At balance snapshot NFT balances query failed due to {str(e)}. Error '
f'is ignored and balance snapshot will still be saved.', )
else:
if len(nft_mapping) != 0:
if str(Location.BLOCKCHAIN) not in balances:
balances[str(Location.BLOCKCHAIN)] = {}
for _, nft_balances in nft_mapping.items():
for balance_entry in nft_balances:
balances[str(Location.BLOCKCHAIN)][Asset(
balance_entry['id'])] = Balance(
amount=FVal(1),
usd_value=balance_entry['usd_price'],
)
balances = account_for_manually_tracked_asset_balances(
db=self.data.db, balances=balances)
# Calculate usd totals
assets_total_balance: DefaultDict[Asset,
Balance] = defaultdict(Balance)
total_usd_per_location: Dict[str, FVal] = {}
for location, asset_balance in balances.items():
total_usd_per_location[location] = ZERO
for asset, balance in asset_balance.items():
assets_total_balance[asset] += balance
total_usd_per_location[location] += balance.usd_value
net_usd = sum((balance.usd_value
for _, balance in assets_total_balance.items()), ZERO)
liabilities_total_usd = sum(
(liability.usd_value for _, liability in liabilities.items()),
ZERO) # noqa: E501
net_usd -= liabilities_total_usd
# Calculate location stats
location_stats: Dict[str, Any] = {}
for location, total_usd in total_usd_per_location.items():
if location == str(Location.BLOCKCHAIN):
total_usd -= liabilities_total_usd
percentage = (total_usd /
net_usd).to_percentage() if net_usd != ZERO else '0%'
location_stats[location] = {
'usd_value': total_usd,
'percentage_of_net_value': percentage,
}
# Calculate 'percentage_of_net_value' per asset
assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = {
asset: balance.to_dict()
for asset, balance in assets_total_balance.items()
}
liabilities_as_dict: Dict[Asset, Dict[str, Any]] = {
asset: balance.to_dict()
for asset, balance in liabilities.items()
}
for asset, balance_dict in assets_total_balance_as_dict.items():
percentage = (balance_dict['usd_value'] / net_usd).to_percentage(
) if net_usd != ZERO else '0%' # noqa: E501
assets_total_balance_as_dict[asset][
'percentage_of_net_value'] = percentage
for asset, balance_dict in liabilities_as_dict.items():
percentage = (balance_dict['usd_value'] / net_usd).to_percentage(
) if net_usd != ZERO else '0%' # noqa: E501
liabilities_as_dict[asset]['percentage_of_net_value'] = percentage
# Compose balances response
result_dict = {
'assets': assets_total_balance_as_dict,
'liabilities': liabilities_as_dict,
'location': location_stats,
'net_usd': net_usd,
}
allowed_to_save = requested_save_data or self.data.db.should_save_balances(
)
if (problem_free or save_despite_errors) and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.db.save_balances_data(data=result_dict,
timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
save_despite_errors=save_despite_errors,
)
return result_dict
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(
settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.ksm_rpc_endpoint is not None:
result, msg = self.chain_manager.set_ksm_rpc_endpoint(
settings.ksm_rpc_endpoint)
if not result:
return False, msg
if settings.dot_rpc_endpoint is not None:
result, msg = self.chain_manager.set_dot_rpc_endpoint(
settings.dot_rpc_endpoint)
if not result:
return False, msg
if settings.btc_derivation_gap_limit is not None:
self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit
if settings.current_price_oracles is not None:
Inquirer().set_oracles_order(settings.current_price_oracles)
if settings.historical_price_oracles is not None:
PriceHistorian().set_oracles_order(
settings.historical_price_oracles)
if settings.active_modules is not None:
self.chain_manager.process_new_modules_list(
settings.active_modules)
self.data.db.set_settings(settings)
return True, ''
def get_settings(self) -> DBSettings:
"""Returns the db settings with a check whether premium is active or not"""
db_settings = self.data.db.get_settings(
have_premium=self.premium is not None)
return db_settings
def setup_exchange(
self,
name: str,
location: Location,
api_key: ApiKey,
api_secret: ApiSecret,
passphrase: Optional[str] = None,
kraken_account_type: Optional['KrakenAccountType'] = None,
PAIRS: Optional[List[str]] = None, # noqa: N803
ftx_subaccount: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret and optionally a passphrase
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
location=location,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
passphrase=passphrase,
ftx_subaccount=ftx_subaccount,
PAIRS=PAIRS,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(
name=name,
location=location,
api_key=api_key,
api_secret=api_secret,
passphrase=passphrase,
kraken_account_type=kraken_account_type,
PAIRS=PAIRS,
ftx_subaccount=ftx_subaccount,
)
return is_success, msg
def remove_exchange(self, name: str,
location: Location) -> Tuple[bool, str]:
if self.exchange_manager.get_exchange(name=name,
location=location) is None:
return False, f'{str(location)} exchange {name} is not registered'
self.exchange_manager.delete_exchange(name=name, location=location)
# Success, remove it also from the DB
self.data.db.remove_exchange(name=name, location=location)
if self.exchange_manager.connected_exchanges.get(location) is None:
# was last exchange of the location type. Delete used query ranges
self.data.db.delete_used_query_range_for_exchange(location)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result[
'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(
NodeName.OWN, None) is not None # noqa : E501
result['last_data_upload_ts'] = Timestamp(
self.premium_sync_manager.last_data_upload_ts) # noqa : E501
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
def _connect_ksm_manager_on_startup(self) -> bool:
return bool(self.data.db.get_blockchain_accounts().ksm)
def _connect_dot_manager_on_startup(self) -> bool:
return bool(self.data.db.get_blockchain_accounts().dot)
def create_oracle_cache(
self,
oracle: HistoricalPriceOracle,
from_asset: Asset,
to_asset: Asset,
purge_old: bool,
) -> None:
"""Creates the cache of the given asset pair from the start of time
until now for the given oracle.
if purge_old is true then any old cache in memory and in a file is purged
May raise:
- RemoteError if there is a problem reaching the oracle
- UnsupportedAsset if any of the two assets is not supported by the oracle
"""
if oracle != HistoricalPriceOracle.CRYPTOCOMPARE:
return # only for cryptocompare for now
self.cryptocompare.create_cache(from_asset, to_asset, purge_old)
class Rotkehlchen(object):
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache = {}
self.connected_exchanges = []
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
loglevel = logging.DEBUG
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.last_data_upload_ts = 0
self.poloniex = None
self.kraken = None
self.bittrex = None
self.binance = None
self.data = DataHandler(self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def initialize_exchanges(self, secret_data):
# initialize exchanges for which we have keys and are not already initialized
if self.kraken is None and 'kraken' in secret_data:
self.kraken = Kraken(
str.encode(secret_data['kraken']['api_key']),
str.encode(secret_data['kraken']['api_secret']), self.data_dir)
self.connected_exchanges.append('kraken')
self.trades_historian.set_exchange('kraken', self.kraken)
if self.poloniex is None and 'poloniex' in secret_data:
self.poloniex = Poloniex(
str.encode(secret_data['poloniex']['api_key']),
str.encode(secret_data['poloniex']['api_secret']),
self.cache_data_filename, self.inquirer, self.data_dir)
self.connected_exchanges.append('poloniex')
self.trades_historian.set_exchange('poloniex', self.poloniex)
if self.bittrex is None and 'bittrex' in secret_data:
self.bittrex = Bittrex(
str.encode(secret_data['bittrex']['api_key']),
str.encode(secret_data['bittrex']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('bittrex')
self.trades_historian.set_exchange('bittrex', self.bittrex)
if self.binance is None and 'binance' in secret_data:
self.binance = Binance(
str.encode(secret_data['binance']['api_key']),
str.encode(secret_data['binance']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('binance')
self.trades_historian.set_exchange('binance', self.binance)
def try_premium_at_start(self, api_key, api_secret, create_new,
sync_approval, user_dir):
"""Check if new user provided api pair or we already got one in the DB"""
if api_key != '':
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
# At this point we are at a new user trying to create an account with
# premium API keys and we failed. But a directory was created. Remove it.
shutil.rmtree(user_dir)
raise AuthenticationError(
'Could not verify keys for the new account. '
'{}'.format(empty_or_error))
else:
# If we got premium initialize it and try to sync with the server
premium_credentials = self.data.db.get_rotkehlchen_premium()
if premium_credentials:
api_key = premium_credentials[0]
api_secret = premium_credentials[1]
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
logger.error(
'The API keys found in the Database are not valid. Perhaps '
'they expired?')
del self.premium
return
else:
# no premium credentials in the DB
return
if self.can_sync_data_from_server():
if sync_approval == 'unknown' and not create_new:
raise PermissionError(
'Rotkehlchen Server has newer version of your DB data. '
'Should we replace local data with the server\'s?')
elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new:
logger.debug('User approved data sync from server')
if self.sync_data_from_server():
if create_new:
# if we succesfully synced data from the server and this is
# a new account, make sure the api keys are properly stored
# in the DB
self.data.db.set_rotkehlchen_premium(
api_key, api_secret)
else:
logger.debug('Could sync data from server but user refused')
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
# unlock or create the DB
self.password = password
user_dir = self.data.unlock(user, password, create_new)
self.try_premium_at_start(api_key, api_secret, create_new,
sync_approval, user_dir)
secret_data = self.data.db.get_exchange_secrets()
self.cache_data_filename = os.path.join(self.data_dir,
'cache_data.json')
historical_data_start = self.data.historical_start_date()
self.trades_historian = TradesHistorian(
self.data_dir,
self.data.db,
self.data.get_eth_accounts(),
historical_data_start,
)
self.price_historian = PriceHistorian(
self.data_dir,
historical_data_start,
)
self.accountant = Accountant(price_historian=self.price_historian,
profit_currency=self.data.main_currency(),
create_csv=True)
self.inquirer = Inquirer(kraken=self.kraken)
self.initialize_exchanges(secret_data)
self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(),
self.data.eth_tokens,
self.data.db.get_owned_tokens(),
self.inquirer, self.args.ethrpc_port)
def set_premium_credentials(self, api_key, api_secret):
if hasattr(self, 'premium'):
valid, empty_or_error = self.premium.set_credentials(
api_key, api_secret)
else:
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if valid:
self.data.set_premium_credentials(api_key, api_secret)
return True, ''
return False, empty_or_error
def maybe_upload_data_to_server(self):
logger.debug('Maybe upload to server')
# upload only if unlocked user has premium
if not hasattr(self, 'premium'):
return
# upload only once per hour
diff = ts_now() - self.last_data_upload_ts
if diff > 3600:
self.upload_data_to_server()
def upload_data_to_server(self):
logger.debug('upload to server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
logger.debug(
'upload to server -- query last metadata error: {}'.format(
result_or_error))
return
logger.debug("CAN_PUSH--> OURS: {} THEIRS: {}".format(
our_hash, result_or_error['data_hash']))
if our_hash == result_or_error['data_hash']:
logger.debug('upload to server -- same hash')
# same hash -- no need to upload anything
return
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts <= result_or_error['last_modify_ts']:
# Server's DB was modified after our local DB
logger.debug("CAN_PUSH -> 3")
logger.debug(
'upload to server -- remote db more recent than local')
return
success, result_or_error = self.premium.upload_data(
data, our_hash, our_last_write_ts, 'zlib')
if not success:
logger.debug(
'upload to server -- upload error: {}'.format(result_or_error))
return
self.last_data_upload_ts = ts_now()
logger.debug('upload to server -- success')
def can_sync_data_from_server(self):
logger.debug('sync data from server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
logger.debug(
'sync data from server-- error: {}'.format(result_or_error))
return False
logger.debug("CAN_PULL--> OURS: {} THEIRS: {}".format(
our_hash, result_or_error['data_hash']))
if our_hash == result_or_error['data_hash']:
logger.debug('sync from server -- same hash')
# same hash -- no need to get anything
return False
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts >= result_or_error['last_modify_ts']:
# Local DB is newer than Server DB
logger.debug(
'sync from server -- local DB more recent than remote')
return False
return True
def sync_data_from_server(self):
success, error_or_result = self.premium.pull_data()
if not success:
logger.debug(
'sync from server -- pulling error {}'.format(error_or_result))
return False
self.data.decompress_and_decrypt_db(self.password,
error_or_result['data'])
return True
def start(self):
return gevent.spawn(self.main_loop)
def main_loop(self):
while True and not self.shutdown_event.is_set():
logger.debug('Main loop start')
if self.poloniex is not None:
self.poloniex.main_logic()
if self.kraken is not None:
self.kraken.main_logic()
self.maybe_upload_data_to_server()
logger.debug('Main loop end')
gevent.sleep(MAIN_LOOP_SECS_DELAY)
def process_history(self, start_ts, end_ts):
history, margin_history, loan_history, asset_movements, eth_transactions = self.trades_historian.get_history(
start_ts=
0, # For entire history processing we need to have full history available
end_ts=ts_now(),
end_at_least_ts=end_ts)
return self.accountant.process_history(start_ts, end_ts, history,
margin_history, loan_history,
asset_movements,
eth_transactions)
def query_fiat_balances(self):
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate
}
return result
def query_balances(self, save_data=False):
balances = {}
for exchange in self.connected_exchanges:
balances[exchange] = getattr(self, exchange).query_balances()
result = self.blockchain.query_balances()['totals']
if result != {}:
balances['blockchain'] = result
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for k, v in combined.items():
net_usd += FVal(v['usd_value'])
stats = {'location': {}, 'net_usd': net_usd}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': (total / net_usd).to_percentage(),
}
for k, v in combined.items():
combined[k]['percentage_of_net_value'] = (v['usd_value'] /
net_usd).to_percentage()
result_dict = merge_dicts(combined, stats)
if save_data:
self.data.save_balances_data(result_dict)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.data.accountant.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency):
with self.lock:
self.data.set_main_currency(currency, self.accountant)
if currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', currency)
def set_settings(self, settings):
with self.lock:
self.data.set_settings(settings)
main_currency = settings['main_currency']
if main_currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
def usd_to_main_currency(self, amount):
main_currency = self.data.main_currency()
if main_currency != 'USD' and not hasattr(self,
'usd_to_main_currency_rate'):
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
return self.usd_to_main_currency_rate * amount
def get_settings(self):
return self.data.settings
def setup_exchange(self, name, api_key, api_secret):
if name not in SUPPORTED_EXCHANGES:
return False, 'Attempted to register unsupported exchange {}'.format(
name)
if getattr(self, name) is not None:
return False, 'Exchange {} is already registered'.format(name)
secret_data = {}
secret_data[name] = {
'api_key': api_key,
'api_secret': api_secret,
}
self.initialize_exchanges(secret_data)
exchange = getattr(self, name)
result, message = exchange.validate_api_key()
if not result:
self.delete_exchange_data(name)
return False, message
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return True, ''
def delete_exchange_data(self, name):
self.connected_exchanges.remove(name)
self.trades_historian.set_exchange(name, None)
delattr(self, name)
setattr(self, name, None)
def remove_exchange(self, name):
if getattr(self, name) is None:
return False, 'Exchange {} is not registered'.format(name)
self.delete_exchange_data(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def shutdown(self):
print("Shutting Down...")
self.shutdown_event.set()
def set(self, *args):
if len(args) < 2:
return ("ERROR: set requires at least two arguments but "
"got: {}".format(args))
if args[0] == 'poloniex':
resp = self.poloniex.set(*args[1:])
else:
return "ERROR: Unrecognized first argument: {}".format(args[0])
self.save_data()
return resp
def test_writting_fetching_external_trades(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
# add 2 trades and check they are in the DB
otc_trade1 = {
'otc_timestamp': '10/03/2018 23:30',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '10',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link',
'otc_notes': 'a note',
}
otc_trade2 = {
'otc_timestamp': '15/03/2018 23:35',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '5',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link 2',
'otc_notes': 'a note 2',
}
trade1, _ = verify_otctrade_data(otc_trade1)
trade2, _ = verify_otctrade_data(otc_trade2)
result, _, = data.add_external_trade(otc_trade1)
assert result
result, _ = data.add_external_trade(otc_trade2)
assert result
result = data.get_external_trades()
assert result[0] == trade1
assert result[1] == trade2
# query trades in period
result = data.get_external_trades(
from_ts=1520553600, # 09/03/2018
to_ts=1520726400, # 11/03/2018
)
assert len(result) == 1
# make sure id is there but do not compare it
assert result[0] == trade1
# query trades only with to_ts
result = data.get_external_trades(
to_ts=1520726400, # 11/03/2018
)
assert len(result) == 1
# make sure id is there but do not compare it
assert result[0] == trade1
# edit a trade and check the edit made it in the DB
otc_trade1['otc_rate'] = '120'
trade1_id = trade1.identifier
otc_trade1['otc_id'] = trade1_id
result, _ = data.edit_external_trade(otc_trade1)
assert result
result = data.get_external_trades()
edited_trade1 = trade1._replace(rate=FVal(120))
assert result[0] == edited_trade1
assert result[1] == trade2
# edit a trade's time (or any other field which makes up the id) and see
# that the ID changes later
new_time_str = '10/03/2018 23:35'
new_timestamp = 1520724900
otc_trade1['otc_timestamp'] = new_time_str
otc_trade1['otc_id'] = edited_trade1.identifier
result, _ = data.edit_external_trade(otc_trade1)
assert result
result = data.get_external_trades()
edited_trade1 = trade1._replace(rate=FVal(120), timestamp=new_timestamp)
assert result[0] == edited_trade1
assert result[1] == trade2
# Here trade is edited succesfully but let's also look in the DB to see
# if the trade is indeed there with a new ID
cursor = data.db.conn.cursor()
results = cursor.execute(
f'SELECT id, time, pair, amount FROM trades where id="{edited_trade1.identifier}";',
)
results = results.fetchall()
assert len(results) == 1
result = results[0]
assert result[0] == edited_trade1.identifier
assert result[1] == new_timestamp
assert result[2] == otc_trade1['otc_pair']
assert result[3] == otc_trade1['otc_amount']
# try to edit a non-existing trade
otc_trade1['otc_rate'] = '160'
otc_trade1['otc_id'] = '5'
result, _ = data.edit_external_trade(otc_trade1)
assert not result
otc_trade1['otc_rate'] = '120'
otc_trade1['otc_id'] = trade1_id
result = data.get_external_trades()
assert result[0] == edited_trade1
assert result[1] == trade2
# try to delete non-existing trade
result, _ = data.delete_external_trade('dasdasd')
assert not result
# delete an external trade
result, _ = data.delete_external_trade(edited_trade1.identifier)
result = data.get_external_trades()
assert result[0] == trade2
def test_add_trades(data_dir, username, caplog):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be logged
data.db.add_trades([trade2, trade3])
assert 'Did not add "buy trade with id a1ed19c8284940b4e59bdac941db2fd3c0ed004ddb10fdd3b9ef0a3a9b2c97bc' in caplog.text # noqa: E501
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2, trade3]
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
self.lock = Semaphore()
self.lock.acquire()
self.premium = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.msg_aggregator = MessagesAggregator()
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: bool,
api_key: ApiKey,
api_secret: ApiSecret,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except AuthenticationError:
# It means that our credentials were not accepted by the server
# or some other error happened
pass
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
# TODO: Once settings returns a named tuple this should go away
msg = 'setting historical_data_start should be a string'
assert isinstance(historical_data_start, str), msg
eth_rpc_endpoint = settings['eth_rpc_endpoint']
msg = 'setting eth_rpc_endpoint should be a string'
assert isinstance(eth_rpc_endpoint, str), msg
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
)
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=Cryptocompare(data_directory=self.data_dir),
)
db_settings = self.data.db.get_settings()
# TODO: Once settings returns a named tuple these should go away
crypto2crypto = db_settings['include_crypto2crypto']
msg = 'settings include_crypto2crypto should be a bool'
assert isinstance(crypto2crypto, bool), msg
taxfree_after_period = db_settings['taxfree_after_period']
msg = 'settings taxfree_after_period should be an int'
assert isinstance(taxfree_after_period, int), msg
include_gas_costs = db_settings['include_gas_costs']
msg = 'settings include_gas_costs should be a bool'
assert isinstance(include_gas_costs, bool), msg
self.accountant = Accountant(
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=crypto2crypto,
taxfree_after_period=taxfree_after_period,
include_gas_costs=include_gas_costs,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
ethchain = Ethchain(eth_rpc_endpoint)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
)
self.user_is_logged_in = True
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
del self.blockchain
self.exchange_manager.delete_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
del self.data_importer
if self.premium is not None:
# For some reason mypy does not see that self.premium is set
del self.premium # type: ignore
self.data.logout()
self.password = ''
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, api_key: ApiKey,
api_secret: ApiSecret) -> None:
"""
Raises IncorrectApiKeyFormat if the given key is not in a proper format
Raises AuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
# For some reason mypy does not see that self.premium is set
self.premium.set_credentials(api_key, api_secret) # type: ignore
else:
self.premium = premium_create_and_verify(api_key, api_secret)
self.data.set_premium_credentials(api_key, api_secret)
def start(self) -> None:
return gevent.spawn(self.main_loop)
def main_loop(self) -> None:
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
log.debug('Main loop start')
self.premium_sync_manager.maybe_upload_data_to_server()
log.debug('Main loop end')
def add_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict:
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict[str, Any]:
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]:
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.track_new_tokens(ethereum_tokens)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]:
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.remove_eth_tokens(ethereum_tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
) = self.trades_historian.get_history(
# For entire history processing we need to have full history available
start_ts=Timestamp(0),
end_ts=ts_now(),
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result, error_or_empty
def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]:
result = {}
balances = self.data.get_fiat_balances()
for currency, str_amount in balances.items():
amount = FVal(str_amount)
usd_rate = Inquirer().query_fiat_pair(currency, A_USD)
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate,
}
return result
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are saved in the DB.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange_balances, _ = exchange.query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange.name] = exchange_balances
result, error_or_empty = self.blockchain.query_balances()
if error_or_empty == '':
balances['blockchain'] = result['totals']
else:
problem_free = False
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
stats: Dict[str, Any] = {
'location': {},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.accountant.events.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency_string: str) -> Tuple[bool, str]:
"""Takes a currency string from the API and sets it as the main currency for rotki
Returns True and empty string for success and False and error string for error
"""
try:
currency = Asset(currency_string)
except UnknownAsset:
msg = f'An unknown asset {currency_string} was given for main currency'
log.critical(msg)
return False, msg
if not currency.is_fiat():
msg = f'A non-fiat asset {currency_string} was given for main currency'
log.critical(msg)
return False, msg
fiat_currency = FiatAsset(currency.identifier)
with self.lock:
self.data.set_main_currency(fiat_currency, self.accountant)
if currency != A_USD:
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
A_USD, currency)
return True, ''
def set_settings(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
log.info('Add new settings')
message = ''
with self.lock:
if 'eth_rpc_endpoint' in settings:
result, msg = self.blockchain.set_eth_rpc_endpoint(
settings['eth_rpc_endpoint'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_endpoint']
message += "\nEthereum RPC endpoint not set: " + msg
if 'main_currency' in settings:
given_symbol = settings['main_currency']
try:
main_currency = Asset(given_symbol)
except UnknownAsset:
return False, f'Unknown fiat currency {given_symbol} provided'
except DeserializationError:
return False, 'Non string type given for fiat currency'
if not main_currency.is_fiat():
msg = (
f'Provided symbol for main currency {given_symbol} is '
f'not a fiat currency')
return False, msg
if main_currency != A_USD:
self.usd_to_main_currency_rate = Inquirer(
).query_fiat_pair(
A_USD,
main_currency,
)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def setup_exchange(
self,
name: str,
api_key: str,
api_secret: str,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret
By default the api keys are always validated unless validate is False.
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return is_success, msg
def remove_exchange(self, name: str) -> Tuple[bool, str]:
if not self.exchange_manager.has_exchange(name):
return False, 'Exchange {} is not registered'.format(name)
self.exchange_manager.delete_exchange(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result['eth_node_connection'] = self.blockchain.ethchain.connected
result[
'history_process_start_ts'] = self.accountant.started_processing_timestamp
result[
'history_process_current_ts'] = self.accountant.currently_processing_timestamp
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
class Rotkehlchen():
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache: ResultCache = dict()
self.premium = None
self.connected_exchanges = []
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.poloniex = None
self.kraken = None
self.bittrex = None
self.bitmex = None
self.binance = None
self.msg_aggregator = MessagesAggregator()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def initialize_exchanges(self, secret_data):
# initialize exchanges for which we have keys and are not already initialized
if self.kraken is None and 'kraken' in secret_data:
self.kraken = Kraken(
api_key=str.encode(secret_data['kraken']['api_key']),
secret=str.encode(secret_data['kraken']['api_secret']),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
usd_eur_price=Inquirer().query_fiat_pair(S_EUR, A_USD),
)
self.connected_exchanges.append('kraken')
self.trades_historian.set_exchange('kraken', self.kraken)
if self.poloniex is None and 'poloniex' in secret_data:
self.poloniex = Poloniex(
api_key=str.encode(secret_data['poloniex']['api_key']),
secret=str.encode(secret_data['poloniex']['api_secret']),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('poloniex')
self.trades_historian.set_exchange('poloniex', self.poloniex)
if self.bittrex is None and 'bittrex' in secret_data:
self.bittrex = Bittrex(
api_key=str.encode(secret_data['bittrex']['api_key']),
secret=str.encode(secret_data['bittrex']['api_secret']),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('bittrex')
self.trades_historian.set_exchange('bittrex', self.bittrex)
if self.binance is None and 'binance' in secret_data:
self.binance = Binance(
api_key=str.encode(secret_data['binance']['api_key']),
secret=str.encode(secret_data['binance']['api_secret']),
data_dir=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('binance')
self.trades_historian.set_exchange('binance', self.binance)
if self.bitmex is None and 'bitmex' in secret_data:
self.bitmex = Bitmex(
api_key=str.encode(secret_data['bitmex']['api_key']),
secret=str.encode(secret_data['bitmex']['api_secret']),
user_directory=self.user_directory,
)
self.connected_exchanges.append('bitmex')
self.trades_historian.set_exchange('bitmex', self.bitmex)
def remove_all_exchanges(self):
if self.kraken is not None:
self.delete_exchange_data('kraken')
if self.poloniex is not None:
self.delete_exchange_data('poloniex')
if self.bittrex is not None:
self.delete_exchange_data('bittrex')
if self.binance is not None:
self.delete_exchange_data('binance')
if self.bitmex is not None:
self.delete_exchange_data('bitmex')
def try_premium_at_start(
self,
api_key: ApiKey,
api_secret: ApiSecret,
username: str,
create_new: bool,
sync_approval: bool,
) -> None:
"""Check if new user provided api pair or we already got one in the DB"""
if api_key != '':
assert create_new, 'We should never get here for an already existing account'
try:
self.premium = premium_create_and_verify(api_key, api_secret)
except (IncorrectApiKeyFormat, AuthenticationError) as e:
log.error('Given API key is invalid')
# At this point we are at a new user trying to create an account with
# premium API keys and we failed. But a directory was created. Remove it.
# But create a backup of it in case something went really wrong
# and the directory contained data we did not want to lose
shutil.move(
self.user_directory,
os.path.join(
self.data_dir,
f'auto_backup_{username}_{ts_now()}',
),
)
shutil.rmtree(self.user_directory)
raise AuthenticationError(
'Could not verify keys for the new account. '
'{}'.format(str(e)), )
# else, if we got premium data in the DB initialize it and try to sync with the server
premium_credentials = self.data.db.get_rotkehlchen_premium()
if premium_credentials:
assert not create_new, 'We should never get here for a new account'
api_key = premium_credentials[0]
api_secret = premium_credentials[1]
try:
self.premium = premium_create_and_verify(api_key, api_secret)
except (IncorrectApiKeyFormat, AuthenticationError) as e:
log.error(
f'Could not authenticate with the rotkehlchen server with '
f'the API keys found in the Database. Error: {str(e)}', )
del self.premium
self.premium = None
if not self.premium:
return
if self.can_sync_data_from_server():
if sync_approval == 'unknown' and not create_new:
log.info('DB data at server newer than local')
raise RotkehlchenPermissionError(
'Rotkehlchen Server has newer version of your DB data. '
'Should we replace local data with the server\'s?', )
elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new:
log.info('User approved data sync from server')
if self.sync_data_from_server():
if create_new:
# if we successfully synced data from the server and this is
# a new account, make sure the api keys are properly stored
# in the DB
self.data.db.set_rotkehlchen_premium(
api_key, api_secret)
else:
log.debug('Could sync data from server but user refused')
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: bool,
api_key: ApiKey,
api_secret: ApiSecret,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_endpoint = settings['eth_rpc_endpoint']
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
msg_aggregator=self.msg_aggregator,
)
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=Cryptocompare(data_directory=self.data_dir),
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'],
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_endpoint)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
)
self.user_is_logged_in = True
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
del self.blockchain
self.remove_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
if self.premium is not None:
del self.premium
self.data.logout()
self.password = ''
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, api_key: ApiKey,
api_secret: ApiSecret) -> None:
"""
Raises IncorrectApiKeyFormat if the given key is not in a proper format
Raises AuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(api_key, api_secret)
else:
self.premium = premium_create_and_verify(api_key, api_secret)
self.data.set_premium_credentials(api_key, api_secret)
def maybe_upload_data_to_server(self) -> None:
# upload only if unlocked user has premium
if self.premium is None:
return
# upload only once per hour
diff = ts_now() - self.last_data_upload_ts
if diff > 3600:
self.upload_data_to_server()
def upload_data_to_server(self) -> None:
log.debug('upload to server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
try:
result = self.premium.query_last_data_metadata()
except RemoteError as e:
log.debug(
'upload to server -- query last metadata failed',
error=str(e),
)
return
log.debug(
'CAN_PUSH',
ours=our_hash,
theirs=result['data_hash'],
)
if our_hash == result['data_hash']:
log.debug('upload to server -- same hash')
# same hash -- no need to upload anything
return
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts <= result['last_modify_ts']:
# Server's DB was modified after our local DB
log.debug("CAN_PUSH -> 3")
log.debug('upload to server -- remote db more recent than local')
return
try:
self.premium.upload_data(
data_blob=data,
our_hash=our_hash,
last_modify_ts=our_last_write_ts,
compression_type='zlib',
)
except RemoteError as e:
log.debug('upload to server -- upload error', error=str(e))
return
# update the last data upload value
self.last_data_upload_ts = ts_now()
self.data.db.update_last_data_upload_ts(self.last_data_upload_ts)
log.debug('upload to server -- success')
def can_sync_data_from_server(self) -> bool:
log.debug('sync data from server -- start')
_, our_hash = self.data.compress_and_encrypt_db(self.password)
try:
result = self.premium.query_last_data_metadata()
except RemoteError as e:
log.debug('sync data from server failed', error=str(e))
return False
if not self.data.db.get_premium_sync():
return False
log.debug(
'CAN_PULL',
ours=our_hash,
theirs=result['data_hash'],
)
if our_hash == result['data_hash']:
log.debug('sync from server -- same hash')
# same hash -- no need to get anything
return False
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts >= result['last_modify_ts']:
# Local DB is newer than Server DB
log.debug('sync from server -- local DB more recent than remote')
return False
return True
def sync_data_from_server(self) -> bool:
try:
result = self.premium.pull_data()
except RemoteError as e:
log.debug('sync from server -- pulling failed.', error=str(e))
return False
self.data.decompress_and_decrypt_db(self.password, result['data'])
return True
def start(self):
return gevent.spawn(self.main_loop)
def main_loop(self):
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
log.debug('Main loop start')
if self.poloniex is not None:
self.poloniex.main_logic()
if self.kraken is not None:
self.kraken.main_logic()
self.maybe_upload_data_to_server()
log.debug('Main loop end')
def add_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict:
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
):
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens: List[str]):
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.track_new_tokens(ethereum_tokens)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens: List[str]):
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.remove_eth_tokens(ethereum_tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(self, start_ts, end_ts):
(
error_or_empty,
history,
margin_history,
loan_history,
asset_movements,
eth_transactions,
) = self.trades_historian.get_history(
start_ts=
0, # For entire history processing we need to have full history available
end_ts=ts_now(),
end_at_least_ts=end_ts,
)
result = self.accountant.process_history(
start_ts,
end_ts,
history,
margin_history,
loan_history,
asset_movements,
eth_transactions,
)
return result, error_or_empty
def query_fiat_balances(self):
log.info('query_fiat_balances called')
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = Inquirer().query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate,
}
return result
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are saved in the DB.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for exchange in self.connected_exchanges:
exchange_balances, _ = getattr(self, exchange).query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange] = exchange_balances
result, error_or_empty = self.blockchain.query_balances()
if error_or_empty == '':
balances['blockchain'] = result['totals']
else:
problem_free = False
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
stats: Dict[str, Any] = {
'location': {},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.data.accountant.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency):
with self.lock:
self.data.set_main_currency(currency, self.accountant)
if currency != 'USD':
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
'USD', currency)
def set_settings(self, settings):
log.info('Add new settings')
message = ''
with self.lock:
if 'eth_rpc_endpoint' in settings:
result, msg = self.blockchain.set_eth_rpc_endpoint(
settings['eth_rpc_endpoint'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_endpoint']
message += "\nEthereum RPC endpoint not set: " + msg
if 'main_currency' in settings:
given_symbol = settings['main_currency']
try:
main_currency = Asset(given_symbol)
except UnknownAsset:
return False, f'Unknown fiat currency {given_symbol} provided'
if not main_currency.is_fiat():
msg = (
f'Provided symbol for main currency {given_symbol} is '
f'not a fiat currency')
return False, msg
if main_currency != A_USD:
self.usd_to_main_currency_rate = Inquirer(
).query_fiat_pair(
'USD',
main_currency.identifier,
)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def setup_exchange(
self,
name: str,
api_key: ApiKey,
api_secret: ApiSecret,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret
By default the api keys are always validated unless validate is False.
"""
log.info('setup_exchange', name=name)
if name not in SUPPORTED_EXCHANGES:
return False, 'Attempted to register unsupported exchange {}'.format(
name)
if getattr(self, name) is not None:
return False, 'Exchange {} is already registered'.format(name)
secret_data = {}
secret_data[name] = {
'api_key': api_key,
'api_secret': api_secret,
}
self.initialize_exchanges(secret_data)
exchange = getattr(self, name)
result, message = exchange.validate_api_key()
if not result:
log.error(
'Failed to validate API key for exchange',
name=name,
error=message,
)
self.delete_exchange_data(name)
return False, message
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return True, ''
def delete_exchange_data(self, name):
self.connected_exchanges.remove(name)
self.trades_historian.set_exchange(name, None)
delattr(self, name)
setattr(self, name, None)
def remove_exchange(self, name):
if getattr(self, name) is None:
return False, 'Exchange {} is not registered'.format(name)
self.delete_exchange_data(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result['eth_node_connection'] = self.blockchain.ethchain.connected
result[
'history_process_current_ts'] = self.accountant.currently_processed_timestamp
return result
def shutdown(self):
self.logout()
self.shutdown_event.set()
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
This runs during backend initialization so it should be as light as possible.
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
self.data_dir.mkdir(parents=True, exist_ok=True)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.main_loop_spawned = False
self.args = args
self.api_task_greenlets: List[gevent.Greenlet] = []
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
# Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver
GlobalDBHandler(data_dir=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir,
coingecko=self.coingecko)
self.assets_updater = AssetsUpdater(self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.task_manager: Optional[TaskManager] = None
self.shutdown_event = gevent.event.Event()
def reset_after_failed_account_creation_or_login(self) -> None:
"""If the account creation or login failed make sure that the rotki instance is clear
Tricky instances are when after either failed premium credentials or user refusal
to sync premium databases we relogged in
"""
self.cryptocompare.db = None
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new,
initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_warning(
'Could not authenticate the rotki premium API keys found in the DB.'
' Has your subscription expired?', )
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
exception_is_error=False,
method=maybe_submit_usage_analytics,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db,
msg_aggregator=self.msg_aggregator)
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
PriceHistorian().set_oracles_order(settings.historical_price_oracles)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
premium=self.premium,
)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
database=self.data.db,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
kusama_manager = SubstrateManager(
chain=SubstrateChain.KUSAMA,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_ksm_manager_on_startup(),
own_rpc_endpoint=settings.ksm_rpc_endpoint,
)
Inquirer().inject_ethereum(ethereum_manager)
Inquirer().set_oracles_order(settings.current_price_oracles)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
kusama_manager=kusama_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
btc_derivation_gap_limit=settings.btc_derivation_gap_limit,
)
self.events_historian = EventsHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.task_manager = TaskManager(
max_tasks_num=DEFAULT_MAX_TASKS_NUM,
greenlet_manager=self.greenlet_manager,
api_task_greenlets=self.api_task_greenlets,
database=self.data.db,
cryptocompare=self.cryptocompare,
premium_sync_manager=self.premium_sync_manager,
chain_manager=self.chain_manager,
exchange_manager=self.exchange_manager,
)
self.greenlet_manager.spawn_and_track(
after_seconds=5,
task_name='periodically_query_icons_until_all_cached',
exception_is_error=False,
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
self.deactivate_premium_status()
self.greenlet_manager.clear()
del self.chain_manager
self.exchange_manager.delete_all_exchanges()
del self.accountant
del self.events_historian
del self.data_importer
self.data.logout()
self.password = ''
self.cryptocompare.unset_database()
# Make sure no messages leak to other user sessions
self.msg_aggregator.consume_errors()
self.msg_aggregator.consume_warnings()
self.task_manager = None
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, credentials: PremiumCredentials) -> None:
"""
Sets the premium credentials for rotki
Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(credentials)
else:
self.premium = premium_create_and_verify(credentials)
self.premium_sync_manager.premium = self.premium
self.accountant.premium = self.premium
self.data.db.set_rotkehlchen_premium(credentials)
def delete_premium_credentials(self) -> Tuple[bool, str]:
"""Deletes the premium credentials for rotki"""
msg = ''
success = self.data.db.del_rotkehlchen_premium()
if success is False:
msg = 'The database was unable to delete the Premium keys for the logged-in user'
self.deactivate_premium_status()
return success, msg
def deactivate_premium_status(self) -> None:
"""Deactivate premium in the current session"""
self.premium = None
self.premium_sync_manager.premium = None
self.chain_manager.deactivate_premium_status()
self.accountant.deactivate_premium_status()
def start(self) -> gevent.Greenlet:
assert not self.main_loop_spawned, 'Tried to spawn the main loop twice'
greenlet = gevent.spawn(self.main_loop)
self.main_loop_spawned = True
return greenlet
def main_loop(self) -> None:
"""rotki main loop that fires often and runs the task manager's scheduler"""
while self.shutdown_event.wait(
timeout=MAIN_LOOP_SECS_DELAY) is not True:
if self.task_manager is not None:
self.task_manager.schedule()
def get_blockchain_account_data(
self,
blockchain: SupportedBlockchain,
) -> Union[List[BlockchainAccountData], Dict[str, Any]]:
account_data = self.data.db.get_blockchain_account_data(blockchain)
if blockchain != SupportedBlockchain.BITCOIN:
return account_data
xpub_data = self.data.db.get_bitcoin_xpub_data()
addresses_to_account_data = {x.address: x for x in account_data}
address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping(
list(addresses_to_account_data.keys()), # type: ignore
)
xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {}
for address, xpub_entry in address_to_xpub_mappings.items():
if xpub_entry not in xpub_mappings:
xpub_mappings[xpub_entry] = []
xpub_mappings[xpub_entry].append(
addresses_to_account_data[address])
data: Dict[str, Any] = {'standalone': [], 'xpubs': []}
# Add xpub data
for xpub_entry in xpub_data:
data_entry = xpub_entry.serialize()
addresses = xpub_mappings.get(xpub_entry, None)
data_entry['addresses'] = addresses if addresses and len(
addresses) != 0 else None
data['xpubs'].append(data_entry)
# Add standalone addresses
for account in account_data:
if account.address not in address_to_xpub_mappings:
data['standalone'].append(account)
return data
def add_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> BlockchainBalancesUpdate:
"""Adds new blockchain accounts
Adds the accounts to the blockchain instance and queries them to get the
updated balances. Also adds them in the DB
May raise:
- EthSyncError from modify_blockchain_account
- InputError if the given accounts list is empty.
- TagConstraintError if any of the given account data contain unknown tags.
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
"""
self.data.db.ensure_tags_exist(
given_data=account_data,
action='adding',
data_type='blockchain accounts',
)
address_type = blockchain.get_address_type()
updated_balances = self.chain_manager.add_blockchain_accounts(
blockchain=blockchain,
accounts=[address_type(entry.address) for entry in account_data],
)
self.data.db.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return updated_balances
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError(
'Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(
self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}', )
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
def remove_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
) -> BlockchainBalancesUpdate:
"""Removes blockchain accounts
Removes the accounts from the blockchain instance and queries them to get
the updated balances. Also removes them from the DB
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- InputError if a non-existing account was given to remove
"""
balances_update = self.chain_manager.remove_blockchain_accounts(
blockchain=blockchain,
accounts=accounts,
)
self.data.db.remove_blockchain_accounts(blockchain, accounts)
return balances_update
def get_history_query_status(self) -> Dict[str, str]:
if self.events_historian.progress < FVal('100'):
processing_state = self.events_historian.processing_state_name
progress = self.events_historian.progress / 2
elif self.accountant.first_processed_timestamp == -1:
processing_state = 'Processing all retrieved historical events'
progress = FVal(50)
else:
processing_state = 'Processing all retrieved historical events'
# start_ts is min of the query start or the first action timestamp since action
# processing can start well before query start to calculate cost basis
start_ts = min(
self.accountant.events.query_start_ts,
self.accountant.first_processed_timestamp,
)
diff = self.accountant.events.query_end_ts - start_ts
progress = 50 + 100 * (
FVal(self.accountant.currently_processing_timestamp - start_ts)
/ FVal(diff) / 2)
return {
'processing_state': str(processing_state),
'total_progress': str(progress)
}
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
defi_events,
ledger_actions,
) = self.events_historian.get_history(
start_ts=start_ts,
end_ts=end_ts,
has_premium=self.premium is not None,
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
defi_events=defi_events,
ledger_actions=ledger_actions,
)
return result, error_or_empty
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade'],
location_actions: TRADES_LIST,
all_actions: TRADES_LIST,
) -> TRADES_LIST:
...
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['asset_movement'],
location_actions: List[AssetMovement],
all_actions: List[AssetMovement],
) -> List[AssetMovement]:
...
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade', 'asset_movement'],
location_actions: Union[TRADES_LIST, List[AssetMovement]],
all_actions: Union[TRADES_LIST, List[AssetMovement]],
) -> Union[TRADES_LIST, List[AssetMovement]]:
"""Take as many actions from location actions and add them to all actions as the limit permits
Returns the modified (or not) all_actions
"""
# If we are already at or above the limit return current actions disregarding this location
actions_mapping = self.actions_per_location[action_type]
current_num_actions = sum(x for _, x in actions_mapping.items())
limit = LIMITS_MAPPING[action_type]
if current_num_actions >= limit:
return all_actions
# Find out how many more actions can we return, and depending on that get
# the number of actions from the location actions and add them to the total
remaining_num_actions = limit - current_num_actions
if remaining_num_actions < 0:
remaining_num_actions = 0
num_actions_to_take = min(len(location_actions), remaining_num_actions)
actions_mapping[location] = num_actions_to_take
all_actions.extend(
location_actions[0:num_actions_to_take]) # type: ignore
return all_actions
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
only_cache: bool,
) -> TRADES_LIST:
"""Queries trades for the given location and time range.
If no location is given then all external, all exchange and DEX trades are queried.
If only_cache is given then only trades cached in the DB are returned.
No service is queried.
DEX Trades are queried only if the user has premium
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
trades: TRADES_LIST
if location is not None:
# clear the trades queried for this location
self.actions_per_location['trade'][location] = 0
trades = self.query_location_trades(from_ts, to_ts, location,
only_cache)
else:
for given_location in ALL_SUPPORTED_EXCHANGES + [
Location.EXTERNAL
]:
# clear the trades queried for this location
self.actions_per_location['trade'][given_location] = 0
trades = self.query_location_trades(from_ts, to_ts,
Location.EXTERNAL, only_cache)
# Look for trades that might be imported from CSV files
for csv_location in EXTERNAL_EXCHANGES:
trades.extend(
self.query_location_trades(
from_ts=from_ts,
to_ts=to_ts,
location=csv_location,
only_cache=only_cache,
))
for exchange in self.exchange_manager.iterate_exchanges():
all_set = {x.identifier for x in trades}
exchange_trades = exchange.query_trade_history(
start_ts=from_ts,
end_ts=to_ts,
only_cache=only_cache,
)
# TODO: Really dirty. Figure out a better way.
# Since some of the trades may already be in the DB if multiple
# keys are used for a single exchange.
exchange_trades = [
x for x in exchange_trades if x.identifier not in all_set
]
if self.premium is None:
trades = self._apply_actions_limit(
location=exchange.location,
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# for all trades we also need the trades from the amm protocols
if self.premium is not None:
for amm_location in AMMTradeLocations:
amm_module_name = cast(AMMTRADE_LOCATION_NAMES,
str(amm_location))
amm_module = self.chain_manager.get_module(amm_module_name)
if amm_module is not None:
trades.extend(
amm_module.get_trades(
addresses=self.chain_manager.
queried_addresses_for_module(
amm_module_name), # noqa: E501
from_timestamp=from_ts,
to_timestamp=to_ts,
only_cache=only_cache,
), )
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
def query_location_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Location,
only_cache: bool,
) -> TRADES_LIST:
location_trades: TRADES_LIST
if location in EXTERNAL_LOCATION:
location_trades = self.data.db.get_trades( # type: ignore # list invariance
from_ts=from_ts,
to_ts=to_ts,
location=location,
)
elif location in AMMTradeLocations:
if self.premium is not None:
amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(location))
amm_module = self.chain_manager.get_module(amm_module_name)
if amm_module is not None:
location_trades = amm_module.get_trades( # type: ignore # list invariance
addresses=self.chain_manager.
queried_addresses_for_module(amm_module_name),
from_timestamp=from_ts,
to_timestamp=to_ts,
only_cache=only_cache,
)
else:
# should only be an exchange
exchanges_list = self.exchange_manager.connected_exchanges.get(
location)
if exchanges_list is None:
logger.warning(
f'Tried to query trades from {str(location)} which is either not an '
f'exchange or not an exchange the user has connected to', )
return []
location_trades = []
for exchange in exchanges_list:
all_set = {x.identifier for x in location_trades}
new_trades = exchange.query_trade_history(
start_ts=from_ts,
end_ts=to_ts,
only_cache=only_cache,
)
# TODO: Really dirty. Figure out a better way.
# Since some of the trades may already be in the DB if multiple
# keys are used for a single exchange.
new_trades = [
x for x in new_trades if x.identifier not in all_set
]
location_trades.extend(new_trades)
trades: TRADES_LIST = []
if self.premium is None:
trades = self._apply_actions_limit(
location=location,
action_type='trade',
location_actions=location_trades,
all_actions=trades,
)
else:
trades = location_trades
return trades
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
ignore_cache: bool = False,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are always saved in the DB,
if it is False then data are saved if self.data.should_save_balances()
is True.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
If ignore_cache is True then all underlying calls that have a cache ignore it
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances: Dict[str, Dict[Asset, Balance]] = {}
problem_free = True
for exchange in self.exchange_manager.iterate_exchanges():
exchange_balances, _ = exchange.query_balances(
ignore_cache=ignore_cache)
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
location_str = str(exchange.location)
if location_str not in balances:
balances[location_str] = exchange_balances
else: # multiple exchange of same type. Combine balances
balances[location_str] = combine_dicts(
balances[location_str],
exchange_balances,
)
liabilities: Dict[Asset, Balance]
try:
blockchain_result = self.chain_manager.query_balances(
blockchain=None,
force_token_detection=ignore_cache,
ignore_cache=ignore_cache,
)
if len(blockchain_result.totals.assets) != 0:
balances[str(
Location.BLOCKCHAIN)] = blockchain_result.totals.assets
liabilities = blockchain_result.totals.liabilities
except (RemoteError, EthSyncError) as e:
problem_free = False
liabilities = {}
log.error(f'Querying blockchain balances failed due to: {str(e)}')
# retrieve loopring balances if module is activated
if self.chain_manager.get_module('loopring'):
loopring_balances = self.chain_manager.get_loopring_balances()
if len(loopring_balances) != 0:
balances[str(Location.LOOPRING)] = loopring_balances
balances = account_for_manually_tracked_balances(db=self.data.db,
balances=balances)
# Calculate usd totals
assets_total_balance: DefaultDict[Asset,
Balance] = defaultdict(Balance)
total_usd_per_location: Dict[str, FVal] = {}
for location, asset_balance in balances.items():
total_usd_per_location[location] = ZERO
for asset, balance in asset_balance.items():
assets_total_balance[asset] += balance
total_usd_per_location[location] += balance.usd_value
net_usd = sum((balance.usd_value
for _, balance in assets_total_balance.items()), ZERO)
liabilities_total_usd = sum(
(liability.usd_value for _, liability in liabilities.items()),
ZERO) # noqa: E501
net_usd -= liabilities_total_usd
# Calculate location stats
location_stats: Dict[str, Any] = {}
for location, total_usd in total_usd_per_location.items():
if location == str(Location.BLOCKCHAIN):
total_usd -= liabilities_total_usd
percentage = (total_usd /
net_usd).to_percentage() if net_usd != ZERO else '0%'
location_stats[location] = {
'usd_value': total_usd,
'percentage_of_net_value': percentage,
}
# Calculate 'percentage_of_net_value' per asset
assets_total_balance_as_dict: Dict[Asset, Dict[str, Any]] = {
asset: balance.to_dict()
for asset, balance in assets_total_balance.items()
}
liabilities_as_dict: Dict[Asset, Dict[str, Any]] = {
asset: balance.to_dict()
for asset, balance in liabilities.items()
}
for asset, balance_dict in assets_total_balance_as_dict.items():
percentage = (balance_dict['usd_value'] / net_usd).to_percentage(
) if net_usd != ZERO else '0%' # noqa: E501
assets_total_balance_as_dict[asset][
'percentage_of_net_value'] = percentage
for asset, balance_dict in liabilities_as_dict.items():
percentage = (balance_dict['usd_value'] / net_usd).to_percentage(
) if net_usd != ZERO else '0%' # noqa: E501
liabilities_as_dict[asset]['percentage_of_net_value'] = percentage
# Compose balances response
result_dict = {
'assets': assets_total_balance_as_dict,
'liabilities': liabilities_as_dict,
'location': location_stats,
'net_usd': net_usd,
}
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.db.save_balances_data(data=result_dict,
timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
return result_dict
def _query_and_populate_exchange_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
all_movements: List[AssetMovement],
exchange: Union[ExchangeInterface, Location],
only_cache: bool,
) -> List[AssetMovement]:
"""Queries exchange for asset movements and adds it to all_movements"""
all_set = {x.identifier for x in all_movements}
if isinstance(exchange, ExchangeInterface):
location = exchange.location
location_movements = exchange.query_deposits_withdrawals(
start_ts=from_ts,
end_ts=to_ts,
only_cache=only_cache,
)
# TODO: Really dirty. Figure out a better way.
# Since some of the asset movements may already be in the DB if multiple
# keys are used for a single exchange.
location_movements = [
x for x in location_movements if x.identifier not in all_set
]
else:
assert isinstance(exchange,
Location), 'only a location should make it here'
assert exchange in EXTERNAL_EXCHANGES, 'only csv supported exchanges should get here' # noqa : E501
location = exchange
# We might have no exchange information but CSV imported information
self.actions_per_location['asset_movement'][location] = 0
location_movements = self.data.db.get_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
location=location,
)
movements: List[AssetMovement] = []
if self.premium is None:
movements = self._apply_actions_limit(
location=location,
action_type='asset_movement',
location_actions=location_movements,
all_actions=all_movements,
)
else:
all_movements.extend(location_movements)
movements = all_movements
return movements
def query_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
only_cache: bool,
) -> List[AssetMovement]:
"""Queries AssetMovements for the given location and time range.
If no location is given then all exchange asset movements are queried.
If only_cache is True then only what is already in the DB is returned.
If the user does not have premium then a limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
movements: List[AssetMovement] = []
if location is not None:
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
if location in EXTERNAL_EXCHANGES:
movements = self._query_and_populate_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=location,
only_cache=only_cache,
)
else:
exchanges_list = self.exchange_manager.connected_exchanges.get(
location)
if exchanges_list is None:
logger.warning(
f'Tried to query deposits/withdrawals from {str(location)} which is '
f'either not an exchange or not an exchange the user has connected to',
)
return []
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
for exchange in exchanges_list:
self._query_and_populate_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
only_cache=only_cache,
)
else:
for exchange_location in ALL_SUPPORTED_EXCHANGES:
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][
exchange_location] = 0
# we may have DB entries due to csv import from supported locations
for external_location in EXTERNAL_EXCHANGES:
movements = self._query_and_populate_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=external_location,
only_cache=only_cache,
)
for exchange in self.exchange_manager.iterate_exchanges():
self._query_and_populate_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
only_cache=only_cache,
)
# return movements with most recent first
movements.sort(key=lambda x: x.timestamp, reverse=True)
return movements
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(
settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.ksm_rpc_endpoint is not None:
result, msg = self.chain_manager.set_ksm_rpc_endpoint(
settings.ksm_rpc_endpoint)
if not result:
return False, msg
if settings.btc_derivation_gap_limit is not None:
self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit
if settings.current_price_oracles is not None:
Inquirer().set_oracles_order(settings.current_price_oracles)
if settings.historical_price_oracles is not None:
PriceHistorian().set_oracles_order(
settings.historical_price_oracles)
if settings.active_modules is not None:
self.chain_manager.process_new_modules_list(
settings.active_modules)
self.data.db.set_settings(settings)
return True, ''
def get_settings(self) -> DBSettings:
"""Returns the db settings with a check whether premium is active or not"""
db_settings = self.data.db.get_settings(
have_premium=self.premium is not None)
return db_settings
def setup_exchange(
self,
name: str,
location: Location,
api_key: ApiKey,
api_secret: ApiSecret,
passphrase: Optional[str] = None,
kraken_account_type: Optional['KrakenAccountType'] = None,
binance_markets: Optional[List[str]] = None,
ftx_subaccount_name: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret and optionally a passphrase
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
location=location,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
passphrase=passphrase,
ftx_subaccount_name=ftx_subaccount_name,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(
name=name,
location=location,
api_key=api_key,
api_secret=api_secret,
passphrase=passphrase,
kraken_account_type=kraken_account_type,
binance_markets=binance_markets,
ftx_subaccount_name=ftx_subaccount_name,
)
return is_success, msg
def remove_exchange(self, name: str,
location: Location) -> Tuple[bool, str]:
if self.exchange_manager.get_exchange(name=name,
location=location) is None:
return False, f'{str(location)} exchange {name} is not registered'
self.exchange_manager.delete_exchange(name=name, location=location)
# Success, remove it also from the DB
self.data.db.remove_exchange(name=name, location=location)
if self.exchange_manager.connected_exchanges.get(location) is None:
# was last exchange of the location type. Delete used query ranges
self.data.db.delete_used_query_range_for_exchange(location)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result[
'eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(
NodeName.OWN, None) is not None # noqa : E501
result['last_data_upload_ts'] = Timestamp(
self.premium_sync_manager.last_data_upload_ts) # noqa : E501
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
def _connect_ksm_manager_on_startup(self) -> bool:
return bool(self.data.db.get_blockchain_accounts().ksm)
def create_oracle_cache(
self,
oracle: HistoricalPriceOracle,
from_asset: Asset,
to_asset: Asset,
purge_old: bool,
) -> None:
"""Creates the cache of the given asset pair from the start of time
until now for the given oracle.
if purge_old is true then any old cache in memory and in a file is purged
May raise:
- RemoteError if there is a problem reaching the oracle
- UnsupportedAsset if any of the two assets is not supported by the oracle
"""
if oracle != HistoricalPriceOracle.CRYPTOCOMPARE:
return # only for cryptocompare for now
self.cryptocompare.create_cache(from_asset, to_asset, purge_old)
def test_add_trades(data_dir, username):
"""Test that adding and retrieving trades from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
trade1 = Trade(
timestamp=1451606400,
location=Location.KRAKEN,
pair='ETH_EUR',
trade_type=TradeType.BUY,
amount=FVal('1.1'),
rate=FVal('10'),
fee=Fee(FVal('0.01')),
fee_currency=A_EUR,
link='',
notes='',
)
trade2 = Trade(
timestamp=1451607500,
location=Location.BINANCE,
pair='BTC_ETH',
trade_type=TradeType.BUY,
amount=FVal('0.00120'),
rate=FVal('10'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
trade3 = Trade(
timestamp=1451608600,
location=Location.COINBASE,
pair='BTC_ETH',
trade_type=TradeType.SELL,
amount=FVal('0.00120'),
rate=FVal('1'),
fee=Fee(FVal('0.001')),
fee_currency=A_ETH,
link='',
notes='',
)
# Add and retrieve the first 2 trades. All should be fine.
data.db.add_trades([trade1, trade2])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2]
# Add the last 2 trades. Since trade2 already exists in the DB it should be
# ignored and a warning should be shown
data.db.add_trades([trade2, trade3])
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 1
returned_trades = data.db.get_trades()
assert returned_trades == [trade1, trade2, trade3]
def populate_db_and_check_for_asset_renaming(
cursor: Cursor,
data: DataHandler,
data_dir: Path,
msg_aggregator: MessagesAggregator,
username: str,
to_rename_asset: str,
renamed_asset: Asset,
target_version: int,
):
# Manually input data to the affected tables.
# timed_balances, multisettings and (external) trades
# At this time point we only have occurence of the to_rename_asset
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value) '
' VALUES(?, ?, ?, ?)',
('1557499129', to_rename_asset, '10.1', '150'),
)
# But add a time point where we got both to_rename_asset and
# renamed_asset. This is to test merging if renaming falls in time where
# both new and old asset had entries
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value) '
' VALUES(?, ?, ?, ?)',
('1558499129', to_rename_asset, '1.1', '15'),
)
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value) '
' VALUES(?, ?, ?, ?)',
('1558499129', renamed_asset.identifier, '2.2', '25'),
)
# Add one different asset for control test
cursor.execute(
'INSERT INTO timed_balances('
' time, currency, amount, usd_value) '
' VALUES(?, ?, ?, ?)',
('1556392121', 'ETH', '5.5', '245'),
)
# Also populate an ignored assets entry
cursor.execute(
'INSERT INTO multisettings(name, value) VALUES(?, ?)',
('ignored_asset', to_rename_asset),
)
cursor.execute(
'INSERT INTO multisettings(name, value) VALUES(?, ?)',
('ignored_asset', 'RDN'),
)
# Finally include it in some trades
cursor.execute(
'INSERT INTO trades('
' time,'
' location,'
' pair,'
' type,'
' amount,'
' rate,'
' fee,'
' fee_currency,'
' link,'
' notes)'
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(
1543298883,
'external',
'ETH_EUR',
'buy',
'100',
'0.5',
'0.1',
'EUR',
'',
'',
),
)
cursor.execute(
'INSERT INTO trades('
' time,'
' location,'
' pair,'
' type,'
' amount,'
' rate,'
' fee,'
' fee_currency,'
' link,'
' notes)'
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(
1563298883,
'kraken',
f'{to_rename_asset}_EUR',
'buy',
'100',
'0.5',
'0.1',
to_rename_asset,
'',
'',
),
)
cursor.execute(
'INSERT INTO trades('
' time,'
' location,'
' pair,'
' type,'
' amount,'
' rate,'
' fee,'
' fee_currency,'
' link,'
' notes)'
'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
(
1564218181,
'binance',
f'{to_rename_asset}_EUR',
'buy',
'100',
'0.5',
'0.1',
'BNB',
'',
'',
),
)
data.db.conn.commit()
# now relogin and check that all tables have appropriate data
del data
data = DataHandler(data_dir, msg_aggregator)
with creation_patch, target_patch(target_version=target_version):
data.unlock(username, '123', create_new=False)
# Check that owned and ignored assets reflect the new state
ignored_assets = data.db.get_ignored_assets()
assert A_RDN in ignored_assets
assert renamed_asset in ignored_assets
owned_assets = data.db.query_owned_assets()
assert A_ETH in owned_assets
assert renamed_asset in owned_assets
# Make sure that the merging of both new and old name entry in same timestamp works
timed_balances = data.db.query_timed_balances(
from_ts=Timestamp(0),
to_ts=Timestamp(2556392121),
asset=renamed_asset,
)
assert len(timed_balances) == 2
assert timed_balances[0].time == 1557499129
assert timed_balances[0].amount == '10.1'
assert timed_balances[0].usd_value == '150'
assert timed_balances[1].time == 1558499129
assert timed_balances[1].amount == '3.3'
assert timed_balances[1].usd_value == '40'
# Assert that trades got renamed properly
cursor = data.db.conn.cursor()
query = ('SELECT id,'
' time,'
' location,'
' pair,'
' type,'
' amount,'
' rate,'
' fee,'
' fee_currency,'
' link,'
' notes FROM trades ORDER BY time ASC;')
results = cursor.execute(query)
trades = []
for result in results:
trades.append({
'id': result[0],
'timestamp': result[1],
'location': result[2],
'pair': result[3],
'trade_type': result[4],
'amount': result[5],
'rate': result[6],
'fee': result[7],
'fee_currency': result[8],
'link': result[9],
'notes': result[10],
})
assert len(trades) == 3
assert trades[0]['fee_currency'] == 'EUR'
assert trades[0]['pair'] == 'ETH_EUR'
assert trades[1]['fee_currency'] == renamed_asset.identifier
assert trades[1]['pair'] == f'{renamed_asset.identifier}_EUR'
assert trades[2]['pair'] == f'{renamed_asset.identifier}_EUR'
assert data.db.get_version() == target_version
def test_add_ethereum_transactions(data_dir, username):
"""Test that adding and retrieving ethereum transactions from the DB works fine.
Also duplicates should be ignored and an error returned
"""
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
tx1 = EthereumTransaction(
tx_hash=b'1',
timestamp=Timestamp(1451606400),
block_number=1,
from_address=ETH_ADDRESS1,
to_address=ETH_ADDRESS3,
value=FVal('2000000'),
gas=FVal('5000000'),
gas_price=FVal('2000000000'),
gas_used=FVal('25000000'),
input_data=MOCK_INPUT_DATA,
nonce=1,
)
tx2 = EthereumTransaction(
tx_hash=b'2',
timestamp=Timestamp(1451706400),
block_number=3,
from_address=ETH_ADDRESS2,
to_address=ETH_ADDRESS3,
value=FVal('4000000'),
gas=FVal('5000000'),
gas_price=FVal('2000000000'),
gas_used=FVal('25000000'),
input_data=MOCK_INPUT_DATA,
nonce=1,
)
tx3 = EthereumTransaction(
tx_hash=b'3',
timestamp=Timestamp(1452806400),
block_number=5,
from_address=ETH_ADDRESS3,
to_address=ETH_ADDRESS1,
value=FVal('1000000'),
gas=FVal('5000000'),
gas_price=FVal('2000000000'),
gas_used=FVal('25000000'),
input_data=MOCK_INPUT_DATA,
nonce=3,
)
# Add and retrieve the first 2 margins. All should be fine.
data.db.add_ethereum_transactions([tx1, tx2], from_etherscan=True)
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_transactions = data.db.get_ethereum_transactions()
assert returned_transactions == [tx1, tx2]
# Add the last 2 transactions. Since tx2 already exists in the DB it should be
# ignored (no errors shown for attempting to add already existing transaction)
data.db.add_ethereum_transactions([tx2, tx3], from_etherscan=True)
errors = msg_aggregator.consume_errors()
warnings = msg_aggregator.consume_warnings()
assert len(errors) == 0
assert len(warnings) == 0
returned_transactions = data.db.get_ethereum_transactions()
assert returned_transactions == [tx1, tx2, tx3]
def test_writting_fetching_data(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
tokens = [A_GNO, A_RDN]
data.write_owned_eth_tokens(tokens)
result = data.db.get_owned_tokens()
assert set(tokens) == set(result)
data.add_blockchain_account(SupportedBlockchain.BITCOIN,
'1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Add a non checksummed address
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0x80b369799104a47e98a553f3329812a44a7facdc',
)
accounts = data.db.get_blockchain_accounts()
assert isinstance(accounts, BlockchainAccounts)
assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
# See that after addition the address has been checksummed
assert set(accounts.eth) == set([
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc'),
])
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Remove non-existing account
with pytest.raises(InputError):
data.remove_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0x136029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Remove existing account
data.remove_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
accounts = data.db.get_blockchain_accounts()
assert accounts.eth == [
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
]
result, _ = data.add_ignored_asset('DAO')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert not result
# Test adding non existing asset
result, msg = data.add_ignored_asset('dsajdhskajdad')
assert not result
assert 'for ignoring is not known/supported' in msg
ignored_assets = data.db.get_ignored_assets()
assert all([isinstance(asset, Asset) for asset in ignored_assets])
assert set(ignored_assets) == set([A_DAO, A_DOGE])
# Test removing asset that is not in the list
result, msg = data.remove_ignored_asset('RDN')
assert 'not in ignored assets' in msg
# Test removing non existing asset
result, msg = data.remove_ignored_asset('dshajdhsjkahdjssad')
assert 'is not known/supported' in msg
assert not result
result, _ = data.remove_ignored_asset('DOGE')
assert result
assert data.db.get_ignored_assets() == ['DAO']
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result['last_write_ts']
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
del result['last_write_ts']
assert result == {
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_endpoint': 'http://localhost:8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'db_version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO,
'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
'main_currency': DEFAULT_MAIN_CURRENCY,
'anonymized_logs': DEFAULT_ANONYMIZED_LOGS,
'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT,
'last_data_upload_ts': 0,
'premium_should_sync': False,
}
# Check setting non-existing settings. Should be ignored
success, msg = data.set_settings({'nonexisting_setting': 1},
accountant=None)
assert success
assert msg != '' and 'nonexisting_setting' in msg
_, msg = data.set_settings(
{
'nonexisting_setting': 1,
'eth_rpc_endpoint': 'http://localhost:8555',
'ui_floating_precision': 3,
},
accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
# Now check nothing funny made it in the db
result = data.db.get_settings()
assert result['eth_rpc_endpoint'] == 'http://localhost:8555'
assert result['ui_floating_precision'] == 3
assert 'nonexisting_setting' not in result
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator)
# Initialize the AssetResolver singleton
AssetResolver(data_directory=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko)
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='periodically_query_icons_until_all_cached',
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.lock.release()
self.shutdown_event = gevent.event.Event()
def reset_after_failed_account_creation_or_login(self) -> None:
"""If the account creation or login failed make sure that the Rotki instance is clear
Tricky instances are when after either failed premium credentials or user refusal
to sync premium databases we relogged in.
"""
self.cryptocompare.db = None
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new, initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data, password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_error(
'Tried to synchronize the database from remote but the local password '
'does not match the one the remote DB has. Please change the password '
'to be the same as the password of the account you want to sync from ',
)
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
method=maybe_submit_usage_analytics,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db, msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db, msg_aggregator=self.msg_aggregator)
historical_data_start = settings.historical_data_start
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=self.cryptocompare,
)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
database=self.data.db,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
Inquirer().inject_ethereum(ethereum_manager)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
)
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
self.greenlet_manager.clear()
del self.chain_manager
self.exchange_manager.delete_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
del self.data_importer
if self.premium is not None:
del self.premium
self.data.logout()
self.password = ''
self.cryptocompare.unset_database()
# Make sure no messages leak to other user sessions
self.msg_aggregator.consume_errors()
self.msg_aggregator.consume_warnings()
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, credentials: PremiumCredentials) -> None:
"""
Sets the premium credentials for Rotki
Raises PremiumAuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(credentials)
else:
self.premium = premium_create_and_verify(credentials)
self.data.db.set_rotkehlchen_premium(credentials)
def delete_premium_credentials(self) -> Tuple[bool, str]:
"""Deletes the premium credentials for Rotki"""
msg = ''
success = self.data.db.del_rotkehlchen_premium()
if success is False:
msg = 'The database was unable to delete the Premium keys for the logged-in user'
self.deactivate_premium_status()
return success, msg
def deactivate_premium_status(self) -> None:
"""Deactivate premium in the current session"""
self.premium = None
self.premium_sync_manager.premium = None
self.chain_manager.deactivate_premium_status()
def start(self) -> gevent.Greenlet:
return gevent.spawn(self.main_loop)
def main_loop(self) -> None:
"""Rotki main loop that fires often and manages many different tasks
Each task remembers the last time it run successfully and know how often it
should run. So each task manages itself.
"""
# super hacky -- organize better when recurring tasks are implemented
# https://github.com/rotki/rotki/issues/1106
xpub_derivation_scheduled = False
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
if self.user_is_logged_in:
log.debug('Main loop start')
self.premium_sync_manager.maybe_upload_data_to_server()
if not xpub_derivation_scheduled:
# 1 minute in the app's startup try to derive new xpub addresses
self.greenlet_manager.spawn_and_track(
after_seconds=60.0,
task_name='Derive new xpub addresses',
method=XpubManager(self.chain_manager).check_for_new_xpub_addresses,
)
xpub_derivation_scheduled = True
log.debug('Main loop end')
def get_blockchain_account_data(
self,
blockchain: SupportedBlockchain,
) -> Union[List[BlockchainAccountData], Dict[str, Any]]:
account_data = self.data.db.get_blockchain_account_data(blockchain)
if blockchain != SupportedBlockchain.BITCOIN:
return account_data
xpub_data = self.data.db.get_bitcoin_xpub_data()
addresses_to_account_data = {x.address: x for x in account_data}
address_to_xpub_mappings = self.data.db.get_addresses_to_xpub_mapping(
list(addresses_to_account_data.keys()), # type: ignore
)
xpub_mappings: Dict['XpubData', List[BlockchainAccountData]] = {}
for address, xpub_entry in address_to_xpub_mappings.items():
if xpub_entry not in xpub_mappings:
xpub_mappings[xpub_entry] = []
xpub_mappings[xpub_entry].append(addresses_to_account_data[address])
data: Dict[str, Any] = {'standalone': [], 'xpubs': []}
# Add xpub data
for xpub_entry in xpub_data:
data_entry = xpub_entry.serialize()
addresses = xpub_mappings.get(xpub_entry, None)
data_entry['addresses'] = addresses if addresses and len(addresses) != 0 else None
data['xpubs'].append(data_entry)
# Add standalone addresses
for account in account_data:
if account.address not in address_to_xpub_mappings:
data['standalone'].append(account)
return data
def add_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> BlockchainBalancesUpdate:
"""Adds new blockchain accounts
Adds the accounts to the blockchain instance and queries them to get the
updated balances. Also adds them in the DB
May raise:
- EthSyncError from modify_blockchain_account
- InputError if the given accounts list is empty.
- TagConstraintError if any of the given account data contain unknown tags.
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
"""
self.data.db.ensure_tags_exist(
given_data=account_data,
action='adding',
data_type='blockchain accounts',
)
address_type = blockchain.get_address_type()
updated_balances = self.chain_manager.add_blockchain_accounts(
blockchain=blockchain,
accounts=[address_type(entry.address) for entry in account_data],
)
self.data.db.add_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return updated_balances
def edit_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
account_data: List[BlockchainAccountData],
) -> None:
"""Edits blockchain accounts
Edits blockchain account data for the given accounts
May raise:
- InputError if the given accounts list is empty or if
any of the accounts to edit do not exist.
- TagConstraintError if any of the given account data contain unknown tags.
"""
# First check for validity of account data addresses
if len(account_data) == 0:
raise InputError('Empty list of blockchain account data to edit was given')
accounts = [x.address for x in account_data]
unknown_accounts = set(accounts).difference(self.chain_manager.accounts.get(blockchain))
if len(unknown_accounts) != 0:
raise InputError(
f'Tried to edit unknown {blockchain.value} '
f'accounts {",".join(unknown_accounts)}',
)
self.data.db.ensure_tags_exist(
given_data=account_data,
action='editing',
data_type='blockchain accounts',
)
# Finally edit the accounts
self.data.db.edit_blockchain_accounts(
blockchain=blockchain,
account_data=account_data,
)
return None
def remove_blockchain_accounts(
self,
blockchain: SupportedBlockchain,
accounts: ListOfBlockchainAddresses,
) -> BlockchainBalancesUpdate:
"""Removes blockchain accounts
Removes the accounts from the blockchain instance and queries them to get
the updated balances. Also removes them from the DB
May raise:
- RemoteError if an external service such as Etherscan is queried and
there is a problem with its query.
- InputError if a non-existing account was given to remove
"""
balances_update = self.chain_manager.remove_blockchain_accounts(
blockchain=blockchain,
accounts=accounts,
)
self.data.db.remove_blockchain_accounts(blockchain, accounts)
return balances_update
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
defi_events,
) = self.trades_historian.get_history(
start_ts=start_ts,
end_ts=end_ts,
has_premium=self.premium is not None,
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
defi_events=defi_events,
)
return result, error_or_empty
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade'],
location_actions: TRADES_LIST,
all_actions: TRADES_LIST,
) -> TRADES_LIST:
...
@overload
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['asset_movement'],
location_actions: List[AssetMovement],
all_actions: List[AssetMovement],
) -> List[AssetMovement]:
...
def _apply_actions_limit(
self,
location: Location,
action_type: Literal['trade', 'asset_movement'],
location_actions: Union[TRADES_LIST, List[AssetMovement]],
all_actions: Union[TRADES_LIST, List[AssetMovement]],
) -> Union[TRADES_LIST, List[AssetMovement]]:
"""Take as many actions from location actions and add them to all actions as the limit permits
Returns the modified (or not) all_actions
"""
# If we are already at or above the limit return current actions disregarding this location
actions_mapping = self.actions_per_location[action_type]
current_num_actions = sum(x for _, x in actions_mapping.items())
limit = LIMITS_MAPPING[action_type]
if current_num_actions >= limit:
return all_actions
# Find out how many more actions can we return, and depending on that get
# the number of actions from the location actions and add them to the total
remaining_num_actions = limit - current_num_actions
if remaining_num_actions < 0:
remaining_num_actions = 0
num_actions_to_take = min(len(location_actions), remaining_num_actions)
actions_mapping[location] = num_actions_to_take
all_actions.extend(location_actions[0:num_actions_to_take]) # type: ignore
return all_actions
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> TRADES_LIST:
"""Queries trades for the given location and time range.
If no location is given then all external, all exchange and DEX trades are queried.
DEX Trades are queried only if the user has premium
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
trades: TRADES_LIST
if location is not None:
trades = self.query_location_trades(from_ts, to_ts, location)
else:
trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL)
# crypto.com is not an API key supported exchange but user can import from CSV
trades.extend(self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM))
for name, exchange in self.exchange_manager.connected_exchanges.items():
exchange_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts)
if self.premium is None:
trades = self._apply_actions_limit(
location=deserialize_location(name),
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# for all trades we also need uniswap trades
if self.premium is not None:
uniswap = self.chain_manager.uniswap
if uniswap is not None:
trades.extend(
uniswap.get_trades(
addresses=self.chain_manager.queried_addresses_for_module('uniswap'),
from_timestamp=from_ts,
to_timestamp=to_ts,
),
)
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
def query_location_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Location,
) -> TRADES_LIST:
# clear the trades queried for this location
self.actions_per_location['trade'][location] = 0
location_trades: TRADES_LIST
if location in (Location.EXTERNAL, Location.CRYPTOCOM):
location_trades = self.data.db.get_trades( # type: ignore # list invariance
from_ts=from_ts,
to_ts=to_ts,
location=location,
)
elif location == Location.UNISWAP:
if self.premium is not None:
uniswap = self.chain_manager.uniswap
if uniswap is not None:
location_trades = uniswap.get_trades( # type: ignore # list invariance
addresses=self.chain_manager.queried_addresses_for_module('uniswap'),
from_timestamp=from_ts,
to_timestamp=to_ts,
)
else:
# should only be an exchange
exchange = self.exchange_manager.get(str(location))
if not exchange:
logger.warn(
f'Tried to query trades from {location} which is either not an '
f'exchange or not an exchange the user has connected to',
)
return []
location_trades = exchange.query_trade_history(start_ts=from_ts, end_ts=to_ts)
trades: TRADES_LIST = []
if self.premium is None:
trades = self._apply_actions_limit(
location=location,
action_type='trade',
location_actions=location_trades,
all_actions=trades,
)
else:
trades = location_trades
return trades
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
ignore_cache: bool = False,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are always saved in the DB,
if it is False then data are saved if self.data.should_save_balances()
is True.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
If ignore_cache is True then all underlying calls that have a cache ignore it
Returns a dictionary with the queried balances.
"""
log.info('query_balances called', requested_save_data=requested_save_data)
balances = {}
problem_free = True
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange_balances, _ = exchange.query_balances(ignore_cache=ignore_cache)
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange.name] = exchange_balances
try:
blockchain_result = self.chain_manager.query_balances(
blockchain=None,
force_token_detection=ignore_cache,
ignore_cache=ignore_cache,
)
serialized_chain_result = blockchain_result.totals.to_dict()
balances['blockchain'] = serialized_chain_result['assets']
except (RemoteError, EthSyncError) as e:
problem_free = False
log.error(f'Querying blockchain balances failed due to: {str(e)}')
balances = account_for_manually_tracked_balances(db=self.data.db, balances=balances)
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value')) for k, v in balances.items()]
liabilities = serialized_chain_result['liabilities'] # atm liabilities only on chain
# calculate net usd value
net_usd = ZERO
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
# subtract liabilities
liabilities_total_usd = sum(x['usd_value'] for _, x in liabilities.items())
net_usd -= liabilities_total_usd
stats: Dict[str, Any] = {
'location': {
},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if name == 'blockchain': # blockchain is the only location with liabilities atm
total -= liabilities_total_usd
if net_usd != ZERO:
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != ZERO:
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
for k, v in liabilities.items():
if net_usd != ZERO:
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
liabilities[k]['percentage_of_net_value'] = percentage
balance_sheet = {
'assets': combined,
'liabilities': liabilities,
}
result_dict = merge_dicts(balance_sheet, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances()
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.db.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
return result_dict
def _query_exchange_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
all_movements: List[AssetMovement],
exchange: Union[ExchangeInterface, Location],
) -> List[AssetMovement]:
if isinstance(exchange, ExchangeInterface):
location = deserialize_location(exchange.name)
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
location_movements = exchange.query_deposits_withdrawals(
start_ts=from_ts,
end_ts=to_ts,
)
else:
assert isinstance(exchange, Location), 'only a location should make it here'
assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here'
location = exchange
# cryptocom has no exchange integration but we may have DB entries
self.actions_per_location['asset_movement'][location] = 0
location_movements = self.data.db.get_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
location=str(location),
)
movements: List[AssetMovement] = []
if self.premium is None:
movements = self._apply_actions_limit(
location=location,
action_type='asset_movement',
location_actions=location_movements,
all_actions=all_movements,
)
else:
all_movements.extend(location_movements)
movements = all_movements
return movements
def query_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> List[AssetMovement]:
"""Queries AssetMovements for the given location and time range.
If no location is given then all exchange asset movements are queried.
If the user does not have premium then a limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
movements: List[AssetMovement] = []
if location is not None:
if location == Location.CRYPTOCOM:
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=Location.CRYPTOCOM,
)
else:
exchange = self.exchange_manager.get(str(location))
if not exchange:
logger.warn(
f'Tried to query deposits/withdrawals from {location} which is either '
f'not at exchange or not an exchange the user has connected to',
)
return []
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
)
else:
# cryptocom has no exchange integration but we may have DB entries due to csv import
movements = self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=Location.CRYPTOCOM,
)
for _, exchange in self.exchange_manager.connected_exchanges.items():
self._query_exchange_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
all_movements=movements,
exchange=exchange,
)
# return movements with most recent first
movements.sort(key=lambda x: x.timestamp, reverse=True)
return movements
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
with self.lock:
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.kraken_account_type is not None:
kraken = self.exchange_manager.get('kraken')
if kraken:
kraken.set_account_type(settings.kraken_account_type) # type: ignore
self.data.db.set_settings(settings)
return True, ''
def get_settings(self) -> DBSettings:
"""Returns the db settings with a check whether premium is active or not"""
db_settings = self.data.db.get_settings(have_premium=self.premium is not None)
return db_settings
def setup_exchange(
self,
name: str,
api_key: ApiKey,
api_secret: ApiSecret,
passphrase: Optional[str] = None,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret and optionally a passphrase
By default the api keys are always validated unless validate is False.
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
passphrase=passphrase,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret, passphrase=passphrase)
return is_success, msg
def remove_exchange(self, name: str) -> Tuple[bool, str]:
if not self.exchange_manager.has_exchange(name):
return False, 'Exchange {} is not registered'.format(name)
self.exchange_manager.delete_exchange(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
self.data.db.delete_used_query_range_for_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result['last_balance_save'] = self.data.db.get_last_balance_save_time()
result['eth_node_connection'] = self.chain_manager.ethereum.web3_mapping.get(NodeName.OWN, None) is not None # noqa : E501
result['history_process_start_ts'] = self.accountant.started_processing_timestamp
result['history_process_current_ts'] = self.accountant.currently_processing_timestamp
result['last_data_upload_ts'] = Timestamp(self.premium_sync_manager.last_data_upload_ts) # noqa : E501
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
def test_writting_fetching_external_trades(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
# add 2 trades and check they are in the DB
trade1 = {
'otc_timestamp': '10/03/2018 23:30',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '10',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link',
'otc_notes': 'a note',
}
trade2 = {
'otc_timestamp': '15/03/2018 23:35',
'otc_pair': 'ETH_EUR',
'otc_type': 'buy',
'otc_amount': '5',
'otc_rate': '100',
'otc_fee': '0.001',
'otc_fee_currency': 'ETH',
'otc_link': 'a link 2',
'otc_notes': 'a note 2',
}
result, _, = data.add_external_trade(trade1)
assert result
result, _ = data.add_external_trade(trade2)
assert result
result = data.get_external_trades()
del result[0]['id']
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# query trades in period
result = data.get_external_trades(
from_ts=1520553600, # 09/03/2018
to_ts=1520726400, # 11/03/2018
)
assert len(result) == 1
del result[0]['id']
assert result[0] == from_otc_trade(trade1)
# query trades only with to_ts
result = data.get_external_trades(
to_ts=1520726400, # 11/03/2018
)
assert len(result) == 1
del result[0]['id']
assert result[0] == from_otc_trade(trade1)
# edit a trade and check the edit made it in the DB
trade1['otc_rate'] = '120'
trade1['otc_id'] = 1
result, _ = data.edit_external_trade(trade1)
assert result
result = data.get_external_trades()
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# try to edit a non-existing trade
trade1['otc_rate'] = '160'
trade1['otc_id'] = 5
result, _ = data.edit_external_trade(trade1)
assert not result
trade1['otc_rate'] = '120'
trade1['otc_id'] = 1
result = data.get_external_trades()
assert result[0] == from_otc_trade(trade1)
del result[1]['id']
assert result[1] == from_otc_trade(trade2)
# try to delete non-existing trade
result, _ = data.delete_external_trade(6)
assert not result
# delete an external trade
result, _ = data.delete_external_trade(1)
result = data.get_external_trades()
del result[0]['id']
assert result[0] == from_otc_trade(trade2)
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise AssertionError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def test_get_latest_location_value_distribution(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
locations = [
LocationData(
time=Timestamp(1451606400),
location='kraken',
usd_value='100',
),
LocationData(
time=Timestamp(1451606400),
location='banks',
usd_value='1000',
),
LocationData(
time=Timestamp(1461606500),
location='poloniex',
usd_value='50',
),
LocationData(
time=Timestamp(1461606500),
location='kraken',
usd_value='200',
),
LocationData(
time=Timestamp(1461606500),
location='banks',
usd_value='50000',
),
LocationData(
time=Timestamp(1491607800),
location='poloniex',
usd_value='100',
),
LocationData(
time=Timestamp(1491607800),
location='kraken',
usd_value='2000',
),
LocationData(
time=Timestamp(1491607800),
location='banks',
usd_value='10000',
),
LocationData(
time=Timestamp(1491607800),
location='blockchain',
usd_value='200000',
),
]
data.db.add_multiple_location_data(locations)
distribution = data.db.get_latest_location_value_distribution()
assert len(distribution) == 4
assert all(entry.time == Timestamp(1491607800) for entry in distribution)
assert distribution[0].location == 'banks'
assert distribution[0].usd_value == '10000'
assert distribution[1].location == 'blockchain'
assert distribution[1].usd_value == '200000'
assert distribution[2].location == 'kraken'
assert distribution[2].usd_value == '2000'
assert distribution[3].location == 'poloniex'
assert distribution[3].usd_value == '100'
