def test_writting_fetching_data(data_dir, username):
msg_aggregator = MessagesAggregator()
data = DataHandler(data_dir, msg_aggregator)
data.unlock(username, '123', create_new=True)
tokens = [A_GNO, A_RDN]
data.write_owned_eth_tokens(tokens)
result = data.db.get_owned_tokens()
assert set(tokens) == set(result)
data.add_blockchain_account(SupportedBlockchain.BITCOIN,
'1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Add a non checksummed address
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0x80b369799104a47e98a553f3329812a44a7facdc',
)
accounts = data.db.get_blockchain_accounts()
assert isinstance(accounts, BlockchainAccounts)
assert accounts.btc == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
# See that after addition the address has been checksummed
assert set(accounts.eth) == set([
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc'),
])
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError): # pylint: disable=no-member
data.add_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Remove non-existing account
with pytest.raises(InputError):
data.remove_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0x136029d76af6fE4A356528e4Dc66B2C18123597D',
)
# Remove existing account
data.remove_blockchain_account(
SupportedBlockchain.ETHEREUM,
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
)
accounts = data.db.get_blockchain_accounts()
assert accounts.eth == [
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
]
result, _ = data.add_ignored_asset('DAO')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert not result
# Test adding non existing asset
result, msg = data.add_ignored_asset('dsajdhskajdad')
assert not result
assert 'for ignoring is not known/supported' in msg
ignored_assets = data.db.get_ignored_assets()
assert all([isinstance(asset, Asset) for asset in ignored_assets])
assert set(ignored_assets) == set([A_DAO, A_DOGE])
# Test removing asset that is not in the list
result, msg = data.remove_ignored_asset('RDN')
assert 'not in ignored assets' in msg
# Test removing non existing asset
result, msg = data.remove_ignored_asset('dshajdhsjkahdjssad')
assert 'is not known/supported' in msg
assert not result
result, _ = data.remove_ignored_asset('DOGE')
assert result
assert data.db.get_ignored_assets() == ['DAO']
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result['last_write_ts']
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
del result['last_write_ts']
assert result == {
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_endpoint': 'http://localhost:8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'db_version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': DEFAULT_INCLUDE_CRYPTO2CRYPTO,
'include_gas_costs': DEFAULT_INCLUDE_GAS_COSTS,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
'main_currency': DEFAULT_MAIN_CURRENCY,
'anonymized_logs': DEFAULT_ANONYMIZED_LOGS,
'date_display_format': DEFAULT_DATE_DISPLAY_FORMAT,
'last_data_upload_ts': 0,
'premium_should_sync': False,
}
# Check setting non-existing settings. Should be ignored
success, msg = data.set_settings({'nonexisting_setting': 1},
accountant=None)
assert success
assert msg != '' and 'nonexisting_setting' in msg
_, msg = data.set_settings(
{
'nonexisting_setting': 1,
'eth_rpc_endpoint': 'http://localhost:8555',
'ui_floating_precision': 3,
},
accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
# Now check nothing funny made it in the db
result = data.db.get_settings()
assert result['eth_rpc_endpoint'] == 'http://localhost:8555'
assert result['ui_floating_precision'] == 3
assert 'nonexisting_setting' not in result
class Rotkehlchen():
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache: ResultCache = dict()
self.premium = None
self.connected_exchanges = []
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.last_data_upload_ts = 0
self.poloniex = None
self.kraken = None
self.bittrex = None
self.bitmex = None
self.binance = None
self.msg_aggregator = MessagesAggregator()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def initialize_exchanges(self, secret_data):
# initialize exchanges for which we have keys and are not already initialized
if self.kraken is None and 'kraken' in secret_data:
self.kraken = Kraken(
api_key=str.encode(secret_data['kraken']['api_key']),
secret=str.encode(secret_data['kraken']['api_secret']),
user_directory=self.user_directory,
usd_eur_price=Inquirer().query_fiat_pair(S_EUR, A_USD),
)
self.connected_exchanges.append('kraken')
self.trades_historian.set_exchange('kraken', self.kraken)
if self.poloniex is None and 'poloniex' in secret_data:
self.poloniex = Poloniex(
api_key=str.encode(secret_data['poloniex']['api_key']),
secret=str.encode(secret_data['poloniex']['api_secret']),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('poloniex')
self.trades_historian.set_exchange('poloniex', self.poloniex)
if self.bittrex is None and 'bittrex' in secret_data:
self.bittrex = Bittrex(
api_key=str.encode(secret_data['bittrex']['api_key']),
secret=str.encode(secret_data['bittrex']['api_secret']),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('bittrex')
self.trades_historian.set_exchange('bittrex', self.bittrex)
if self.binance is None and 'binance' in secret_data:
self.binance = Binance(
api_key=str.encode(secret_data['binance']['api_key']),
secret=str.encode(secret_data['binance']['api_secret']),
data_dir=self.user_directory,
msg_aggregator=self.msg_aggregator,
)
self.connected_exchanges.append('binance')
self.trades_historian.set_exchange('binance', self.binance)
if self.bitmex is None and 'bitmex' in secret_data:
self.bitmex = Bitmex(
api_key=str.encode(secret_data['bitmex']['api_key']),
secret=str.encode(secret_data['bitmex']['api_secret']),
user_directory=self.user_directory,
)
self.connected_exchanges.append('bitmex')
self.trades_historian.set_exchange('bitmex', self.bitmex)
def remove_all_exchanges(self):
if self.kraken is not None:
self.delete_exchange_data('kraken')
if self.poloniex is not None:
self.delete_exchange_data('poloniex')
if self.bittrex is not None:
self.delete_exchange_data('bittrex')
if self.binance is not None:
self.delete_exchange_data('binance')
if self.bitmex is not None:
self.delete_exchange_data('bitmex')
def try_premium_at_start(self, api_key, api_secret, username, create_new,
sync_approval):
"""Check if new user provided api pair or we already got one in the DB"""
if api_key != '':
assert create_new, 'We should never get here for an already existing account'
try:
self.premium = premium_create_and_verify(api_key, api_secret)
except (IncorrectApiKeyFormat, AuthenticationError) as e:
log.error('Given API key is invalid')
# At this point we are at a new user trying to create an account with
# premium API keys and we failed. But a directory was created. Remove it.
# But create a backup of it in case something went really wrong
# and the directory contained data we did not want to lose
shutil.move(
self.user_directory,
os.path.join(
self.data_dir,
f'auto_backup_{username}_{ts_now()}',
),
)
shutil.rmtree(self.user_directory)
raise AuthenticationError(
'Could not verify keys for the new account. '
'{}'.format(str(e)), )
# else, if we got premium initialize it and try to sync with the server
premium_credentials = self.data.db.get_rotkehlchen_premium()
if premium_credentials:
api_key = premium_credentials[0]
api_secret = premium_credentials[1]
try:
self.premium = premium_create_and_verify(api_key, api_secret)
except (IncorrectApiKeyFormat, AuthenticationError) as e:
log.error(
f'Could not authenticate with the rotkehlchen server with '
f'the API keys found in the Database. Error: {str(e)}', )
del self.premium
self.premium = None
if not self.premium:
return
if self.can_sync_data_from_server():
if sync_approval == 'unknown' and not create_new:
log.info('DB data at server newer than local')
raise RotkehlchenPermissionError(
'Rotkehlchen Server has newer version of your DB data. '
'Should we replace local data with the server\'s?', )
elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new:
log.info('User approved data sync from server')
if self.sync_data_from_server():
if create_new:
# if we successfully synced data from the server and this is
# a new account, make sure the api keys are properly stored
# in the DB
self.data.db.set_rotkehlchen_premium(
api_key, api_secret)
else:
log.debug('Could sync data from server but user refused')
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: bool,
api_key: ApiKey,
api_secret: ApiSecret,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
historical_data_start=historical_data_start,
msg_aggregator=self.msg_aggregator,
)
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=Cryptocompare(data_directory=self.data_dir),
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'],
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
)
self.user_is_logged_in = True
def logout(self):
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
del self.blockchain
self.blockchain = None
self.remove_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
self.accountant = None
del self.trades_historian
self.trades_historian = None
if self.premium is not None:
del self.premium
self.premium = None
self.data.logout()
self.password = None
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, api_key: ApiKey,
api_secret: ApiSecret) -> None:
"""
Raises IncorrectApiKeyFormat if the given key is not in a proper format
Raises AuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
self.premium.set_credentials(api_key, api_secret)
else:
self.premium = premium_create_and_verify(api_key, api_secret)
self.data.set_premium_credentials(api_key, api_secret)
def maybe_upload_data_to_server(self):
# upload only if unlocked user has premium
if self.premium is None:
return
# upload only once per hour
diff = ts_now() - self.last_data_upload_ts
if diff > 3600:
self.upload_data_to_server()
def upload_data_to_server(self) -> None:
log.debug('upload to server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
try:
result = self.premium.query_last_data_metadata()
except RemoteError as e:
log.debug(
'upload to server -- query last metadata failed',
error=str(e),
)
return
log.debug(
'CAN_PUSH',
ours=our_hash,
theirs=result['data_hash'],
)
if our_hash == result['data_hash']:
log.debug('upload to server -- same hash')
# same hash -- no need to upload anything
return
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts <= result['last_modify_ts']:
# Server's DB was modified after our local DB
log.debug("CAN_PUSH -> 3")
log.debug('upload to server -- remote db more recent than local')
return
try:
self.premium.upload_data(
data_blob=data,
our_hash=our_hash,
last_modify_ts=our_last_write_ts,
compression_type='zlib',
)
except RemoteError as e:
log.debug('upload to server -- upload error', error=str(e))
return
self.last_data_upload_ts = ts_now()
log.debug('upload to server -- success')
def can_sync_data_from_server(self) -> bool:
log.debug('sync data from server -- start')
_, our_hash = self.data.compress_and_encrypt_db(self.password)
try:
result = self.premium.query_last_data_metadata()
except RemoteError as e:
log.debug('sync data from server failed', error=str(e))
return False
log.debug(
'CAN_PULL',
ours=our_hash,
theirs=result['data_hash'],
)
if our_hash == result['data_hash']:
log.debug('sync from server -- same hash')
# same hash -- no need to get anything
return False
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts >= result['last_modify_ts']:
# Local DB is newer than Server DB
log.debug('sync from server -- local DB more recent than remote')
return False
return True
def sync_data_from_server(self) -> bool:
try:
result = self.premium.pull_data()
except RemoteError as e:
log.debug('sync from server -- pulling failed.', error=str(e))
return False
self.data.decompress_and_decrypt_db(self.password, result)
return True
def start(self):
return gevent.spawn(self.main_loop)
def main_loop(self):
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
log.debug('Main loop start')
if self.poloniex is not None:
self.poloniex.main_logic()
if self.kraken is not None:
self.kraken.main_logic()
self.maybe_upload_data_to_server()
log.debug('Main loop end')
def add_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict:
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
):
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens: List[str]):
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.track_new_tokens(ethereum_tokens)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens: List[str]):
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.remove_eth_tokens(ethereum_tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(self, start_ts, end_ts):
(
error_or_empty,
history,
margin_history,
loan_history,
asset_movements,
eth_transactions,
) = self.trades_historian.get_history(
start_ts=
0, # For entire history processing we need to have full history available
end_ts=ts_now(),
end_at_least_ts=end_ts,
)
result = self.accountant.process_history(
start_ts,
end_ts,
history,
margin_history,
loan_history,
asset_movements,
eth_transactions,
)
return result, error_or_empty
def query_fiat_balances(self):
log.info('query_fiat_balances called')
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = Inquirer().query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate,
}
return result
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are saved in the DB.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for exchange in self.connected_exchanges:
exchange_balances, _ = getattr(self, exchange).query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange] = exchange_balances
result, error_or_empty = self.blockchain.query_balances()
if error_or_empty == '':
balances['blockchain'] = result['totals']
else:
problem_free = False
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
stats: Dict[str, Any] = {
'location': {},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.data.accountant.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency):
with self.lock:
self.data.set_main_currency(currency, self.accountant)
if currency != 'USD':
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
'USD', currency)
def set_settings(self, settings):
log.info('Add new settings')
message = ''
with self.lock:
if 'eth_rpc_port' in settings:
result, msg = self.blockchain.set_eth_rpc_port(
settings['eth_rpc_port'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_port']
message += "\nEthereum RPC port not set: " + msg
if 'main_currency' in settings:
main_currency = settings['main_currency']
if main_currency != 'USD':
self.usd_to_main_currency_rate = Inquirer(
).query_fiat_pair(
'USD',
main_currency,
)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def usd_to_main_currency(self, amount):
main_currency = self.data.main_currency()
if main_currency != 'USD' and not hasattr(self,
'usd_to_main_currency_rate'):
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
'USD', main_currency)
return self.usd_to_main_currency_rate * amount
def setup_exchange(
self,
name: str,
api_key: ApiKey,
api_secret: ApiSecret,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret
By default the api keys are always validated unless validate is False.
"""
log.info('setup_exchange', name=name)
if name not in SUPPORTED_EXCHANGES:
return False, 'Attempted to register unsupported exchange {}'.format(
name)
if getattr(self, name) is not None:
return False, 'Exchange {} is already registered'.format(name)
secret_data = {}
secret_data[name] = {
'api_key': api_key,
'api_secret': api_secret,
}
self.initialize_exchanges(secret_data)
exchange = getattr(self, name)
result, message = exchange.validate_api_key()
if not result:
log.error(
'Failed to validate API key for exchange',
name=name,
error=message,
)
self.delete_exchange_data(name)
return False, message
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return True, ''
def delete_exchange_data(self, name):
self.connected_exchanges.remove(name)
self.trades_historian.set_exchange(name, None)
delattr(self, name)
setattr(self, name, None)
def remove_exchange(self, name):
if getattr(self, name) is None:
return False, 'Exchange {} is not registered'.format(name)
self.delete_exchange_data(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result['eth_node_connection'] = self.blockchain.ethchain.connected
result[
'history_process_current_ts'] = self.accountant.currently_processed_timestamp
return result
def shutdown(self):
self.logout()
self.shutdown_event.set()
class Rotkehlchen():
def __init__(self, args: argparse.Namespace) -> None:
self.lock = Semaphore()
self.lock.acquire()
self.premium = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.msg_aggregator = MessagesAggregator()
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: bool,
api_key: ApiKey,
api_secret: ApiSecret,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except AuthenticationError:
# It means that our credentials were not accepted by the server
# or some other error happened
pass
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
# TODO: Once settings returns a named tuple this should go away
msg = 'setting historical_data_start should be a string'
assert isinstance(historical_data_start, str), msg
eth_rpc_endpoint = settings['eth_rpc_endpoint']
msg = 'setting eth_rpc_endpoint should be a string'
assert isinstance(eth_rpc_endpoint, str), msg
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
)
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=Cryptocompare(data_directory=self.data_dir),
)
db_settings = self.data.db.get_settings()
# TODO: Once settings returns a named tuple these should go away
crypto2crypto = db_settings['include_crypto2crypto']
msg = 'settings include_crypto2crypto should be a bool'
assert isinstance(crypto2crypto, bool), msg
taxfree_after_period = db_settings['taxfree_after_period']
msg = 'settings taxfree_after_period should be an int'
assert isinstance(taxfree_after_period, int), msg
include_gas_costs = db_settings['include_gas_costs']
msg = 'settings include_gas_costs should be a bool'
assert isinstance(include_gas_costs, bool), msg
self.accountant = Accountant(
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=crypto2crypto,
taxfree_after_period=taxfree_after_period,
include_gas_costs=include_gas_costs,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
ethchain = Ethchain(eth_rpc_endpoint)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
)
self.user_is_logged_in = True
def logout(self) -> None:
if not self.user_is_logged_in:
return
user = self.data.username
log.info(
'Logging out user',
user=user,
)
del self.blockchain
self.exchange_manager.delete_all_exchanges()
# Reset rotkehlchen logger to default
LoggingSettings(anonymized_logs=DEFAULT_ANONYMIZED_LOGS)
del self.accountant
del self.trades_historian
del self.data_importer
if self.premium is not None:
# For some reason mypy does not see that self.premium is set
del self.premium # type: ignore
self.data.logout()
self.password = ''
self.user_is_logged_in = False
log.info(
'User successfully logged out',
user=user,
)
def set_premium_credentials(self, api_key: ApiKey,
api_secret: ApiSecret) -> None:
"""
Raises IncorrectApiKeyFormat if the given key is not in a proper format
Raises AuthenticationError if the given key is rejected by the Rotkehlchen server
"""
log.info('Setting new premium credentials')
if self.premium is not None:
# For some reason mypy does not see that self.premium is set
self.premium.set_credentials(api_key, api_secret) # type: ignore
else:
self.premium = premium_create_and_verify(api_key, api_secret)
self.data.set_premium_credentials(api_key, api_secret)
def start(self) -> None:
return gevent.spawn(self.main_loop)
def main_loop(self) -> None:
while self.shutdown_event.wait(MAIN_LOOP_SECS_DELAY) is not True:
log.debug('Main loop start')
self.premium_sync_manager.maybe_upload_data_to_server()
log.debug('Main loop end')
def add_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict:
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(
self,
blockchain: SupportedBlockchain,
account: BlockchainAddress,
) -> Dict[str, Any]:
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]:
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.track_new_tokens(ethereum_tokens)
except (InputError, EthSyncError) as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens: List[str]) -> Dict[str, Any]:
ethereum_tokens = [
EthereumToken(identifier=identifier) for identifier in tokens
]
try:
new_data = self.blockchain.remove_eth_tokens(ethereum_tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[Dict[str, Any], str]:
(
error_or_empty,
history,
loan_history,
asset_movements,
eth_transactions,
) = self.trades_historian.get_history(
# For entire history processing we need to have full history available
start_ts=Timestamp(0),
end_ts=ts_now(),
)
result = self.accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result, error_or_empty
def query_fiat_balances(self) -> Dict[Asset, Dict[str, FVal]]:
result = {}
balances = self.data.get_fiat_balances()
for currency, str_amount in balances.items():
amount = FVal(str_amount)
usd_rate = Inquirer().query_fiat_pair(currency, A_USD)
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate,
}
return result
def query_balances(
self,
requested_save_data: bool = False,
timestamp: Timestamp = None,
) -> Dict[str, Any]:
"""Query all balances rotkehlchen can see.
If requested_save_data is True then the data are saved in the DB.
If timestamp is None then the current timestamp is used.
If a timestamp is given then that is the time that the balances are going
to be saved in the DB
Returns a dictionary with the queried balances.
"""
log.info('query_balances called',
requested_save_data=requested_save_data)
balances = {}
problem_free = True
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange_balances, _ = exchange.query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not isinstance(exchange_balances, dict):
problem_free = False
else:
balances[exchange.name] = exchange_balances
result, error_or_empty = self.blockchain.query_balances()
if error_or_empty == '':
balances['blockchain'] = result['totals']
else:
problem_free = False
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for _, v in combined.items():
net_usd += FVal(v['usd_value'])
stats: Dict[str, Any] = {
'location': {},
'net_usd': net_usd,
}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
if net_usd != FVal(0):
percentage = (total / net_usd).to_percentage()
else:
percentage = '0%'
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': percentage,
}
for k, v in combined.items():
if net_usd != FVal(0):
percentage = (v['usd_value'] / net_usd).to_percentage()
else:
percentage = '0%'
combined[k]['percentage_of_net_value'] = percentage
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
if not timestamp:
timestamp = Timestamp(int(time.time()))
self.data.save_balances_data(data=result_dict, timestamp=timestamp)
log.debug('query_balances data saved')
else:
log.debug(
'query_balances data not saved',
allowed_to_save=allowed_to_save,
problem_free=problem_free,
)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.accountant.events.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency_string: str) -> Tuple[bool, str]:
"""Takes a currency string from the API and sets it as the main currency for rotki
Returns True and empty string for success and False and error string for error
"""
try:
currency = Asset(currency_string)
except UnknownAsset:
msg = f'An unknown asset {currency_string} was given for main currency'
log.critical(msg)
return False, msg
if not currency.is_fiat():
msg = f'A non-fiat asset {currency_string} was given for main currency'
log.critical(msg)
return False, msg
fiat_currency = FiatAsset(currency.identifier)
with self.lock:
self.data.set_main_currency(fiat_currency, self.accountant)
if currency != A_USD:
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
A_USD, currency)
return True, ''
def set_settings(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
log.info('Add new settings')
message = ''
with self.lock:
if 'eth_rpc_endpoint' in settings:
result, msg = self.blockchain.set_eth_rpc_endpoint(
settings['eth_rpc_endpoint'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_endpoint']
message += "\nEthereum RPC endpoint not set: " + msg
if 'main_currency' in settings:
given_symbol = settings['main_currency']
try:
main_currency = Asset(given_symbol)
except UnknownAsset:
return False, f'Unknown fiat currency {given_symbol} provided'
except DeserializationError:
return False, 'Non string type given for fiat currency'
if not main_currency.is_fiat():
msg = (
f'Provided symbol for main currency {given_symbol} is '
f'not a fiat currency')
return False, msg
if main_currency != A_USD:
self.usd_to_main_currency_rate = Inquirer(
).query_fiat_pair(
A_USD,
main_currency,
)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def setup_exchange(
self,
name: str,
api_key: str,
api_secret: str,
) -> Tuple[bool, str]:
"""
Setup a new exchange with an api key and an api secret
By default the api keys are always validated unless validate is False.
"""
is_success, msg = self.exchange_manager.setup_exchange(
name=name,
api_key=api_key,
api_secret=api_secret,
database=self.data.db,
)
if is_success:
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return is_success, msg
def remove_exchange(self, name: str) -> Tuple[bool, str]:
if not self.exchange_manager.has_exchange(name):
return False, 'Exchange {} is not registered'.format(name)
self.exchange_manager.delete_exchange(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def query_periodic_data(self) -> Dict[str, Union[bool, Timestamp]]:
"""Query for frequently changing data"""
result: Dict[str, Union[bool, Timestamp]] = {}
if self.user_is_logged_in:
result[
'last_balance_save'] = self.data.db.get_last_balance_save_time(
)
result['eth_node_connection'] = self.blockchain.ethchain.connected
result[
'history_process_start_ts'] = self.accountant.started_processing_timestamp
result[
'history_process_current_ts'] = self.accountant.currently_processing_timestamp
return result
def shutdown(self) -> None:
self.logout()
self.shutdown_event.set()
class Rotkehlchen(object):
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache: typing.ResultsCacheMap = dict()
self.connected_exchanges = []
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.last_data_upload_ts = 0
self.poloniex = None
self.kraken = None
self.bittrex = None
self.binance = None
self.data = DataHandler(self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def initialize_exchanges(self, secret_data):
# initialize exchanges for which we have keys and are not already initialized
if self.kraken is None and 'kraken' in secret_data:
self.kraken = Kraken(
str.encode(secret_data['kraken']['api_key']),
str.encode(secret_data['kraken']['api_secret']), self.data_dir)
self.connected_exchanges.append('kraken')
self.trades_historian.set_exchange('kraken', self.kraken)
if self.poloniex is None and 'poloniex' in secret_data:
self.poloniex = Poloniex(
str.encode(secret_data['poloniex']['api_key']),
str.encode(secret_data['poloniex']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('poloniex')
self.trades_historian.set_exchange('poloniex', self.poloniex)
if self.bittrex is None and 'bittrex' in secret_data:
self.bittrex = Bittrex(
str.encode(secret_data['bittrex']['api_key']),
str.encode(secret_data['bittrex']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('bittrex')
self.trades_historian.set_exchange('bittrex', self.bittrex)
if self.binance is None and 'binance' in secret_data:
self.binance = Binance(
str.encode(secret_data['binance']['api_key']),
str.encode(secret_data['binance']['api_secret']),
self.inquirer, self.data_dir)
self.connected_exchanges.append('binance')
self.trades_historian.set_exchange('binance', self.binance)
def try_premium_at_start(self, api_key, api_secret, create_new,
sync_approval, user_dir):
"""Check if new user provided api pair or we already got one in the DB"""
if api_key != '':
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
# At this point we are at a new user trying to create an account with
# premium API keys and we failed. But a directory was created. Remove it.
shutil.rmtree(user_dir)
raise AuthenticationError(
'Could not verify keys for the new account. '
'{}'.format(empty_or_error))
else:
# If we got premium initialize it and try to sync with the server
premium_credentials = self.data.db.get_rotkehlchen_premium()
if premium_credentials:
api_key = premium_credentials[0]
api_secret = premium_credentials[1]
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if not valid:
logger.error(
'The API keys found in the Database are not valid. Perhaps '
'they expired?')
del self.premium
return
else:
# no premium credentials in the DB
return
if self.can_sync_data_from_server():
if sync_approval == 'unknown' and not create_new:
raise PermissionError(
'Rotkehlchen Server has newer version of your DB data. '
'Should we replace local data with the server\'s?')
elif sync_approval == 'yes' or sync_approval == 'unknown' and create_new:
logger.debug('User approved data sync from server')
if self.sync_data_from_server():
if create_new:
# if we successfully synced data from the server and this is
# a new account, make sure the api keys are properly stored
# in the DB
self.data.db.set_rotkehlchen_premium(
api_key, api_secret)
else:
logger.debug('Could sync data from server but user refused')
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
# unlock or create the DB
self.password = password
user_dir = self.data.unlock(user, password, create_new)
self.try_premium_at_start(api_key, api_secret, create_new,
sync_approval, user_dir)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
self.data_dir,
self.data.db,
self.data.get_eth_accounts(),
historical_data_start,
)
self.price_historian = PriceHistorian(
self.data_dir,
historical_data_start,
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
price_historian=self.price_historian,
profit_currency=self.data.main_currency(),
user_directory=user_dir,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
)
self.inquirer = Inquirer(kraken=self.kraken)
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
all_eth_tokens=self.data.eth_tokens,
owned_eth_tokens=self.data.db.get_owned_tokens(),
inquirer=self.inquirer,
ethchain=ethchain,
)
def set_premium_credentials(self, api_key, api_secret):
if hasattr(self, 'premium'):
valid, empty_or_error = self.premium.set_credentials(
api_key, api_secret)
else:
self.premium, valid, empty_or_error = premium_create_and_verify(
api_key, api_secret)
if valid:
self.data.set_premium_credentials(api_key, api_secret)
return True, ''
return False, empty_or_error
def maybe_upload_data_to_server(self):
logger.debug('Maybe upload to server')
# upload only if unlocked user has premium
if not hasattr(self, 'premium'):
return
# upload only once per hour
diff = ts_now() - self.last_data_upload_ts
if diff > 3600:
self.upload_data_to_server()
def upload_data_to_server(self):
logger.debug('upload to server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
logger.debug(
'upload to server -- query last metadata error: {}'.format(
result_or_error))
return
logger.debug("CAN_PUSH--> OURS: {} THEIRS: {}".format(
our_hash, result_or_error['data_hash']))
if our_hash == result_or_error['data_hash']:
logger.debug('upload to server -- same hash')
# same hash -- no need to upload anything
return
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts <= result_or_error['last_modify_ts']:
# Server's DB was modified after our local DB
logger.debug("CAN_PUSH -> 3")
logger.debug(
'upload to server -- remote db more recent than local')
return
success, result_or_error = self.premium.upload_data(
data, our_hash, our_last_write_ts, 'zlib')
if not success:
logger.debug(
'upload to server -- upload error: {}'.format(result_or_error))
return
self.last_data_upload_ts = ts_now()
logger.debug('upload to server -- success')
def can_sync_data_from_server(self):
logger.debug('sync data from server -- start')
data, our_hash = self.data.compress_and_encrypt_db(self.password)
success, result_or_error = self.premium.query_last_data_metadata()
if not success:
logger.debug(
'sync data from server-- error: {}'.format(result_or_error))
return False
logger.debug("CAN_PULL--> OURS: {} THEIRS: {}".format(
our_hash, result_or_error['data_hash']))
if our_hash == result_or_error['data_hash']:
logger.debug('sync from server -- same hash')
# same hash -- no need to get anything
return False
our_last_write_ts = self.data.db.get_last_write_ts()
if our_last_write_ts >= result_or_error['last_modify_ts']:
# Local DB is newer than Server DB
logger.debug(
'sync from server -- local DB more recent than remote')
return False
return True
def sync_data_from_server(self):
success, error_or_result = self.premium.pull_data()
if not success:
logger.debug(
'sync from server -- pulling error {}'.format(error_or_result))
return False
self.data.decompress_and_decrypt_db(self.password,
error_or_result['data'])
return True
def start(self):
return gevent.spawn(self.main_loop)
def main_loop(self):
while True and not self.shutdown_event.is_set():
logger.debug('Main loop start')
if self.poloniex is not None:
self.poloniex.main_logic()
if self.kraken is not None:
self.kraken.main_logic()
self.maybe_upload_data_to_server()
logger.debug('Main loop end')
gevent.sleep(MAIN_LOOP_SECS_DELAY)
def add_blockchain_account(self, blockchain, account):
try:
new_data = self.blockchain.add_blockchain_account(
blockchain, account)
except InputError as e:
return simple_result(False, str(e))
self.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(self, blockchain, account):
try:
new_data = self.blockchain.remove_blockchain_account(
blockchain, account)
except InputError as e:
return simple_result(False, str(e))
self.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_owned_eth_tokens(self, tokens):
try:
new_data = self.blockchain.track_new_tokens(tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens):
try:
new_data = self.blockchain.remove_eth_tokens(tokens)
except InputError as e:
return simple_result(False, str(e))
self.data.write_owned_eth_tokens(self.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def process_history(self, start_ts, end_ts):
(
error_or_empty, history, margin_history, loan_history,
asset_movements, eth_transactions
) = self.trades_historian.get_history(
start_ts=
0, # For entire history processing we need to have full history available
end_ts=ts_now(),
end_at_least_ts=end_ts)
result = self.accountant.process_history(start_ts, end_ts, history,
margin_history, loan_history,
asset_movements,
eth_transactions)
return result, error_or_empty
def query_fiat_balances(self):
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate
}
return result
def query_balances(self, requested_save_data=False):
balances = {}
problem_free = True
for exchange in self.connected_exchanges:
exchange_balances, msg = getattr(self, exchange).query_balances()
# If we got an error, disregard that exchange but make sure we don't save data
if not exchange_balances:
problem_free = False
else:
balances[exchange] = exchange_balances
result = self.blockchain.query_balances()['totals']
if result != {}:
balances['blockchain'] = result
result = self.query_fiat_balances()
if result != {}:
balances['banks'] = result
combined = combine_stat_dicts([v for k, v in balances.items()])
total_usd_per_location = [(k, dict_get_sumof(v, 'usd_value'))
for k, v in balances.items()]
# calculate net usd value
net_usd = FVal(0)
for k, v in combined.items():
net_usd += FVal(v['usd_value'])
stats = {'location': {}, 'net_usd': net_usd}
for entry in total_usd_per_location:
name = entry[0]
total = entry[1]
stats['location'][name] = {
'usd_value': total,
'percentage_of_net_value': (total / net_usd).to_percentage(),
}
for k, v in combined.items():
combined[k]['percentage_of_net_value'] = (v['usd_value'] /
net_usd).to_percentage()
result_dict = merge_dicts(combined, stats)
allowed_to_save = requested_save_data or self.data.should_save_balances(
)
if problem_free and allowed_to_save:
self.data.save_balances_data(result_dict)
# After adding it to the saved file we can overlay additional data that
# is not required to be saved in the history file
try:
details = self.data.accountant.details
for asset, (tax_free_amount, average_buy_value) in details.items():
if asset not in result_dict:
continue
result_dict[asset]['tax_free_amount'] = tax_free_amount
result_dict[asset]['average_buy_value'] = average_buy_value
current_price = result_dict[asset]['usd_value'] / result_dict[
asset]['amount']
if average_buy_value != FVal(0):
result_dict[asset]['percent_change'] = (
((current_price - average_buy_value) /
average_buy_value) * 100)
else:
result_dict[asset]['percent_change'] = 'INF'
except AttributeError:
pass
return result_dict
def set_main_currency(self, currency):
with self.lock:
self.data.set_main_currency(currency, self.accountant)
if currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', currency)
def set_settings(self, settings):
message = ''
with self.lock:
if 'eth_rpc_port' in settings:
result, msg = self.blockchain.set_eth_rpc_port(
settings['eth_rpc_port'])
if not result:
# Don't save it in the DB
del settings['eth_rpc_port']
message += "\nEthereum RPC port not set: " + msg
if 'main_currency' in settings:
main_currency = settings['main_currency']
if main_currency != 'USD':
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
res, msg = self.accountant.customize(settings)
if not res:
message += '\n' + msg
return False, message
_, msg, = self.data.set_settings(settings, self.accountant)
if msg != '':
message += '\n' + msg
# Always return success here but with a message
return True, message
def usd_to_main_currency(self, amount):
main_currency = self.data.main_currency()
if main_currency != 'USD' and not hasattr(self,
'usd_to_main_currency_rate'):
self.usd_to_main_currency_rate = query_fiat_pair(
'USD', main_currency)
return self.usd_to_main_currency_rate * amount
def setup_exchange(self, name, api_key, api_secret):
if name not in SUPPORTED_EXCHANGES:
return False, 'Attempted to register unsupported exchange {}'.format(
name)
if getattr(self, name) is not None:
return False, 'Exchange {} is already registered'.format(name)
secret_data = {}
secret_data[name] = {
'api_key': api_key,
'api_secret': api_secret,
}
self.initialize_exchanges(secret_data)
exchange = getattr(self, name)
result, message = exchange.validate_api_key()
if not result:
self.delete_exchange_data(name)
return False, message
# Success, save the result in the DB
self.data.db.add_exchange(name, api_key, api_secret)
return True, ''
def delete_exchange_data(self, name):
self.connected_exchanges.remove(name)
self.trades_historian.set_exchange(name, None)
delattr(self, name)
setattr(self, name, None)
def remove_exchange(self, name):
if getattr(self, name) is None:
return False, 'Exchange {} is not registered'.format(name)
self.delete_exchange_data(name)
# Success, remove it also from the DB
self.data.db.remove_exchange(name)
return True, ''
def shutdown(self):
print("Shutting Down...")
self.shutdown_event.set()
def set(self, *args):
if len(args) < 2:
return ("ERROR: set requires at least two arguments but "
"got: {}".format(args))
if args[0] == 'poloniex':
resp = self.poloniex.set(*args[1:])
else:
return "ERROR: Unrecognized first argument: {}".format(args[0])
self.save_data()
return resp
def test_writting_fetching_data(data_dir, username):
data = DataHandler(data_dir)
data.unlock(username, '123', create_new=True)
tokens = ['GNO', 'RDN']
data.write_owned_eth_tokens(tokens)
result = data.db.get_owned_tokens()
assert set(tokens) == set(result)
data.add_blockchain_account('BTC', '1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS')
data.add_blockchain_account('ETH',
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
data.add_blockchain_account('ETH',
'0x80b369799104a47e98a553f3329812a44a7facdc')
accounts = data.db.get_blockchain_accounts()
assert len(accounts) == 2
assert accounts['BTC'] == ['1CB7Pbji3tquDtMRp8mBkerimkFzWRkovS']
assert set(accounts['ETH']) == set([
'0xd36029d76af6fE4A356528e4Dc66B2C18123597D',
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
])
# Add existing account should fail
with pytest.raises(sqlcipher.IntegrityError):
data.add_blockchain_account(
'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
# Remove non-existing account
with pytest.raises(InputError):
data.remove_blockchain_account(
'ETH', '0x136029d76af6fE4A356528e4Dc66B2C18123597D')
# Remove existing account
data.remove_blockchain_account(
'ETH', '0xd36029d76af6fE4A356528e4Dc66B2C18123597D')
accounts = data.db.get_blockchain_accounts()
assert accounts['ETH'] == [
to_checksum_address('0x80b369799104a47e98a553f3329812a44a7facdc')
]
result, _ = data.add_ignored_asset('DAO')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert result
result, _ = data.add_ignored_asset('DOGE')
assert not result
assert set(data.db.get_ignored_assets()) == set(['DAO', 'DOGE'])
result, _ = data.remove_ignored_asset('XXX')
assert not result
result, _ = data.remove_ignored_asset('DOGE')
assert result
assert data.db.get_ignored_assets() == ['DAO']
# With nothing inserted in settings make sure default values are returned
result = data.db.get_settings()
last_write_diff = ts_now() - result['last_write_ts']
# make sure last_write was within 3 secs
assert last_write_diff >= 0 and last_write_diff < 3
del result['last_write_ts']
assert result == {
'historical_data_start': DEFAULT_START_DATE,
'eth_rpc_port': '8545',
'ui_floating_precision': DEFAULT_UI_FLOATING_PRECISION,
'db_version': ROTKEHLCHEN_DB_VERSION,
'include_crypto2crypto': True,
'taxfree_after_period': YEAR_IN_SECONDS,
'balance_save_frequency': DEFAULT_BALANCE_SAVE_FREQUENCY,
'last_balance_save': 0,
}
# Check setting non-existing settings. Should be ignored
_, msg = data.set_settings({'nonexisting_setting': 1}, accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
_, msg = data.set_settings(
{
'nonexisting_setting': 1,
'eth_rpc_port': '8555',
'ui_floating_precision': 3,
},
accountant=None)
assert msg != '' and 'nonexisting_setting' in msg
# Now check nothing funny made it in the db
result = data.db.get_settings()
assert result['eth_rpc_port'] == '8555'
assert result['ui_floating_precision'] == 3
assert 'nonexisting_setting' not in result
