def query_balances(self, **kwargs: Any) -> ExchangeQueryBalances:
assets_balance: Dict[Asset, Balance] = {}
try:
resp_info = self._api_query('get', 'user/balance')
except RemoteError as e:
msg = (
'ICONOMI API request failed. Could not reach ICONOMI due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
if resp_info['currency'] != 'USD':
raise RemoteError('Iconomi API did not return values in USD')
for balance_info in resp_info['assetList']:
ticker = balance_info['ticker']
try:
asset = asset_from_iconomi(ticker)
# There seems to be a bug in the ICONOMI API regarding balance_info['value'].
# The value is supposed to be in USD, but is actually returned
# in EUR. So let's use the Inquirer for now.
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing ICONOMI balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
try:
amount = deserialize_asset_amount(balance_info['balance'])
except DeserializationError as e:
self.msg_aggregator.add_warning(
f'Skipping iconomi balance entry {balance_info} due to {str(e)}',
)
continue
assets_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
except (UnknownAsset, UnsupportedAsset) as e:
asset_tag = 'unknown' if isinstance(e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} ICONOMI asset {ticker}. '
f' Ignoring its balance query.',
)
continue
for balance_info in resp_info['daaList']:
ticker = balance_info['ticker']
self.msg_aggregator.add_warning(
f'Found unsupported ICONOMI strategy {ticker}. '
f' Ignoring its balance query.',
)
return assets_balance, ''
def _get_known_asset_price(
known_assets: Set[EthereumToken],
unknown_assets: Set[UnknownEthereumToken],
) -> AssetPrice:
"""Get the tokens prices via Inquirer
Given an asset, if `find_usd_price()` returns zero, it will be added
into `unknown_assets`.
"""
asset_price: AssetPrice = {}
for known_asset in known_assets:
asset_usd_price = Inquirer().find_usd_price(known_asset)
if asset_usd_price != Price(ZERO):
asset_price[known_asset.ethereum_address] = asset_usd_price
else:
unknown_asset = UnknownEthereumToken(
ethereum_address=known_asset.ethereum_address,
symbol=known_asset.identifier,
name=known_asset.name,
decimals=known_asset.decimals,
)
unknown_assets.add(unknown_asset)
return asset_price
def query_balances(self) -> ExchangeQueryBalances:
try:
resp = self.api_query_dict('returnCompleteBalances',
{"account": "all"})
except RemoteError as e:
msg = ('Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e))
log.error(msg)
return None, msg
assets_balance: Dict[Asset, Balance] = {}
for poloniex_asset, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if available != ZERO or on_orders != ZERO:
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
log.error(
f'Unexpected poloniex asset type. Expected string '
f' but got {type(poloniex_asset)}', )
self.msg_aggregator.add_error(
'Found poloniex asset entry with non-string type. '
' Ignoring its balance query.', )
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing poloniex balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
amount = available + on_orders
usd_value = amount * usd_price
assets_balance[asset] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Poloniex balance query',
sensitive_log=True,
currency=asset,
amount=amount,
usd_value=usd_value,
)
return assets_balance, ''
def query_balances(self) -> ExchangeQueryBalances:
try:
balances = self._private_api_query('balances')
balances.extend(self._private_api_query('balances/earn'))
except (GeminiPermissionError, RemoteError) as e:
msg = f'Gemini API request failed. {str(e)}'
log.error(msg)
return None, msg
returned_balances: DefaultDict[Asset, Balance] = defaultdict(Balance)
for entry in balances:
try:
balance_type = entry['type']
if balance_type == 'exchange':
amount = deserialize_asset_amount(entry['amount'])
else: # should be 'Earn'
amount = deserialize_asset_amount(entry['balance'])
# ignore empty balances
if amount == ZERO:
continue
asset = asset_from_gemini(entry['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing gemini {balance_type} balance result due to '
f'inability to query USD price: {str(e)}. Skipping balance entry',
)
continue
returned_balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini {balance_type} balance result with unsupported '
f'asset {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error processing a gemini {balance_type} balance. Check logs '
f'for details. Ignoring it.', )
log.error(
f'Error processing a gemini {balance_type} balance',
error=msg,
)
continue
return returned_balances, ''
def _query_cross_collateral_futures_balances(
self,
balances: DefaultDict[Asset, Balance],
) -> DefaultDict[Asset, Balance]:
"""Queries binance collateral future balances and if any found adds them to `balances`
May raise:
- RemoteError
"""
futures_response = self.api_query_dict('sapi', 'futures/loan/wallet')
try:
cross_collaterals = futures_response['crossCollaterals']
for entry in cross_collaterals:
amount = deserialize_asset_amount(entry['locked'])
if amount == ZERO:
continue
try:
asset = asset_from_binance(entry['collateralCoin'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported {self.name} asset {e.asset_name}. '
f'Ignoring its futures balance query.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown {self.name} asset {e.asset_name}. '
f'Ignoring its futures balance query.',
)
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found {self.name} asset with non-string type '
f'{type(entry["asset"])}. Ignoring its futures balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except KeyError as e:
self.msg_aggregator.add_error(
f'At {self.name} futures balance query did not find expected key '
f'{str(e)}. Skipping futures query...',
)
return balances
def usd_to_main_currency(self, amount):
main_currency = self.data.main_currency()
if main_currency != 'USD' and not hasattr(self,
'usd_to_main_currency_rate'):
self.usd_to_main_currency_rate = Inquirer().query_fiat_pair(
'USD', main_currency)
return self.usd_to_main_currency_rate * amount
def __init__(self, args):
self.lock = Semaphore()
self.lock.acquire()
self.results_cache: ResultCache = dict()
self.premium = None
self.connected_exchanges = []
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise ValueError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('zerorpc').setLevel(logging.CRITICAL)
logging.getLogger('zerorpc.channel').setLevel(logging.CRITICAL)
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.last_data_upload_ts = 0
self.poloniex = None
self.kraken = None
self.bittrex = None
self.bitmex = None
self.binance = None
self.msg_aggregator = MessagesAggregator()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def _query_margined_futures_balances(
self,
api_type: Literal['fapi', 'dapi'],
balances: DefaultDict[Asset, Balance],
) -> DefaultDict[Asset, Balance]:
try:
response = self.api_query_list(api_type, 'balance')
except BinancePermissionError as e:
log.warning(
f'Insufficient permission to query {self.name} {api_type} balances.'
f'Skipping query. Response details: {str(e)}', )
return balances
try:
for entry in response:
amount = FVal(entry['balance'])
if amount == ZERO:
continue
try:
asset = asset_from_binance(entry['asset'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported {self.name} asset {e.asset_name}. '
f'Ignoring its margined futures balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown {self.name} asset {e.asset_name}. '
f'Ignoring its margined futures balance query.', )
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found {self.name} asset with non-string type '
f'{type(entry["asset"])}. Ignoring its margined futures balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance entry due to inability to '
f'query USD price: {str(e)}. Skipping margined futures balance entry',
)
continue
balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except KeyError as e:
self.msg_aggregator.add_error(
f'At {self.name} margined futures balance query did not find '
f'expected key {str(e)}. Skipping margined futures query...', )
return balances
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
create_new=create_new,
sync_approval=sync_approval,
)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
historical_data_start=historical_data_start,
)
self.inquirer = Inquirer(data_dir=self.data_dir, kraken=self.kraken)
price_historian = PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
inquirer=self.inquirer,
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
price_historian=price_historian,
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'],
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
all_eth_tokens=self.data.eth_tokens,
owned_eth_tokens=self.data.db.get_owned_tokens(),
inquirer=self.inquirer,
ethchain=ethchain,
)
def modify_eth_account(
self,
account: ChecksumEthAddress,
append_or_remove: str,
add_or_sub: Callable[[FVal, FVal], FVal],
) -> None:
"""Either appends or removes an ETH acccount.
Call with 'append', operator.add to add the account
Call with 'remove', operator.sub to remove the account
May raise:
- Input error if the given_account is not a valid ETH address
- BadFunctionCallOutput if a token is queried from a local chain
and the chain is not synced
- RemoteError if there is a problem with a query to an external
service such as Etherscan or cryptocompare
"""
eth_usd_price = Inquirer().find_usd_price(A_ETH)
remove_with_populated_balance = (append_or_remove == 'remove'
and len(self.balances.eth) != 0)
# Query the balance of the account except for the case when it's removed
# and there is no other account in the balances
if append_or_remove == 'append' or remove_with_populated_balance:
amount = self.ethereum.get_eth_balance(account)
usd_value = amount * eth_usd_price
if append_or_remove == 'append':
self.accounts.eth.append(account)
self.balances.eth[account] = EthereumAccountBalance(
start_eth_amount=amount,
start_eth_usd_value=usd_value,
)
elif append_or_remove == 'remove':
if account not in self.accounts.eth:
raise InputError('Tried to remove a non existing ETH account')
self.accounts.eth.remove(account)
if account in self.balances.eth:
del self.balances.eth[account]
else:
raise AssertionError(
'Programmer error: Should be append or remove')
if len(self.balances.eth) == 0:
# If the last account was removed balance should be 0
self.totals[A_ETH].amount = ZERO
self.totals[A_ETH].usd_value = ZERO
else:
self.totals[A_ETH].amount = add_or_sub(self.totals[A_ETH].amount,
amount)
self.totals[A_ETH].usd_value = add_or_sub(
self.totals[A_ETH].usd_value, usd_value)
action = AccountAction.APPEND if append_or_remove == 'append' else AccountAction.REMOVE
self._query_ethereum_tokens(
action=action,
given_accounts=[account],
)
def query_balances(self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
balances = self._private_api_query('balances')
except (GeminiPermissionError, RemoteError) as e:
msg = f'Gemini API request failed. {str(e)}'
log.error(msg)
return None, msg
returned_balances: Dict[Asset, Dict[str, Any]] = {}
for entry in balances:
try:
amount = deserialize_asset_amount(entry['amount'])
# ignore empty balances
if amount == ZERO:
continue
asset = Asset(entry['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing gemini balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry',
)
continue
returned_balances[asset] = {
'amount': amount,
'usd_value': amount * usd_price,
}
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unknown asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found gemini balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a gemini balance. Check logs '
'for details. Ignoring it.',
)
log.error(
'Error processing a gemini balance',
error=msg,
)
continue
return returned_balances, ''
def query_balances(self) -> ExchangeQueryBalances:
try:
resp = self._api_query('wallet/all_balances', paginate=False)
except RemoteError as e:
msg = f'FTX API request failed. Could not reach FTX due to {str(e)}'
log.error(msg)
return None, msg
# flatten the list that maps accounts to balances
balances = [x for _, bal in resp.items() for x in bal]
# extract the balances and aggregate them
returned_balances: DefaultDict[Asset, Balance] = defaultdict(Balance)
for balance_info in balances:
try:
amount = deserialize_asset_amount(balance_info['total'])
# ignore empty balances. FTX returns zero for some coins
if amount == ZERO:
continue
asset = asset_from_ftx(balance_info['coin'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing FTX balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
returned_balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found FTX balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing an FTX account balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing an FTX balance',
error=msg,
)
continue
return dict(returned_balances), ''
def query_balances(self) -> Tuple[Optional[dict], str]:
try:
old_balances = self.api_query('Balance', req={})
except RemoteError as e:
if "Missing key: 'result'" in str(e):
# handle https://github.com/rotki/rotki/issues/946
old_balances = {}
else:
msg = ('Kraken API request failed. Could not reach kraken due '
'to {}'.format(e))
log.error(msg)
return None, msg
balances = {}
for k, v in old_balances.items():
v = FVal(v)
if v == FVal(0):
continue
try:
our_asset = asset_from_kraken(k)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported/unknown kraken asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found kraken asset with non-string type {type(k)}. '
f' Ignoring its balance query.', )
continue
entry = {}
entry['amount'] = v
if k == 'KFEE':
# There is no price value for KFEE. TODO: Shouldn't we then just skip the balance?
entry['usd_value'] = ZERO
else:
try:
usd_price = Inquirer().find_usd_price(our_asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing kraken balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
entry['usd_value'] = FVal(v * usd_price)
balances[our_asset] = entry
log.debug(
'kraken balance query result',
sensitive_log=True,
currency=our_asset,
amount=entry['amount'],
usd_value=entry['usd_value'],
)
return balances, ''
def query_balances(self) -> Tuple[Optional[dict], str]:
self.first_connection()
try:
# account data returns a dict as per binance docs
account_data = self.api_query_dict('account')
except RemoteError as e:
msg = ('Binance API request failed. Could not reach binance due '
'to {}'.format(e))
log.error(msg)
return None, msg
returned_balances = {}
for entry in account_data['balances']:
amount = entry['free'] + entry['locked']
if amount == FVal(0):
continue
try:
asset = asset_from_binance(entry['asset'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported binance asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown binance asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found binance asset with non-string type {type(entry["asset"])}. '
f' Ignoring its balance query.', )
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing binance balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
balance = {}
balance['amount'] = amount
balance['usd_value'] = FVal(amount * usd_price)
returned_balances[asset] = balance
log.debug(
'binance balance query result',
sensitive_log=True,
asset=asset,
amount=amount,
usd_value=balance['usd_value'],
)
return returned_balances, ''
def query_balances(self) -> ExchangeQueryBalances:
try:
wallets, _, _ = self._api_query('wallets')
# asset_wallets = self._api_query('asset-wallets')
fiat_wallets, _, _ = self._api_query('fiatwallets')
except RemoteError as e:
msg = f'Failed to query Bitpanda balances. {str(e)}'
return None, msg
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
wallets_len = len(wallets)
for idx, entry in enumerate(wallets + fiat_wallets):
if idx < wallets_len:
symbol_key = 'cryptocoin_symbol'
else:
symbol_key = 'fiat_symbol'
try:
amount = deserialize_asset_amount(
entry['attributes']['balance'])
asset = asset_from_bitpanda(entry['attributes'][symbol_key])
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported/unknown Bitpanda asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing Bitpanda balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing bitpanda balance',
entry=entry,
error=msg,
)
continue
if amount == ZERO:
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing Bitpanda balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return dict(assets_balance), ''
def get_fiat_exchange_rates(currencies: Optional[List[Asset]]) -> Response:
if currencies is not None and len(currencies) == 0:
return api_response(
wrap_in_fail_result('Empy list of currencies provided'),
status_code=HTTPStatus.BAD_REQUEST,
)
rates = Inquirer().get_fiat_usd_exchange_rates(currencies)
res = process_result(rates)
return api_response(_wrap_in_ok_result(res), status_code=HTTPStatus.OK)
def _query_vault_data(
self,
identifier: int,
owner: ChecksumEthAddress,
urn: ChecksumEthAddress,
ilk: bytes,
) -> Optional[MakerDAOVault]:
collateral_type = ilk.split(b'\0', 1)[0].decode()
asset = COLLATERAL_TYPE_MAPPING.get(collateral_type, None)
if asset is None:
self.msg_aggregator.add_warning(
f'Detected vault with collateral_type {collateral_type}. That '
f'is not yet supported by rotki. Skipping...', )
return None
result = MAKERDAO_VAT.call(self.ethereum, 'urns', arguments=[ilk, urn])
# also known as ink in their contract
collateral_amount = FVal(result[0] / WAD)
normalized_debt = result[1] # known as art in their contract
result = MAKERDAO_VAT.call(self.ethereum, 'ilks', arguments=[ilk])
rate = result[1] # Accumulated Rates
spot = FVal(result[2]) # Price with Safety Margin
# How many DAI owner needs to pay back to the vault
debt_value = FVal(((normalized_debt / WAD) * rate) / RAY)
result = MAKERDAO_SPOT.call(self.ethereum, 'ilks', arguments=[ilk])
mat = result[1]
liquidation_ratio = FVal(mat / RAY)
price = FVal((spot / RAY) * liquidation_ratio)
self.usd_price[asset.identifier] = price
collateral_value = FVal(price * collateral_amount)
if debt_value == 0:
collateralization_ratio = None
else:
collateralization_ratio = FVal(collateral_value /
debt_value).to_percentage(2)
collateral_usd_value = price * collateral_amount
if collateral_amount == 0:
liquidation_price = None
else:
liquidation_price = (debt_value *
liquidation_ratio) / collateral_amount
dai_usd_price = Inquirer().find_usd_price(A_DAI)
return MakerDAOVault(
identifier=identifier,
owner=owner,
collateral_type=collateral_type,
collateral_asset=asset,
collateral=Balance(collateral_amount, collateral_usd_value),
debt=Balance(debt_value, dai_usd_price * debt_value),
liquidation_ratio=liquidation_ratio,
collateralization_ratio=collateralization_ratio,
liquidation_price=liquidation_price,
urn=urn,
stability_fee=self.get_stability_fee(ilk),
)
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator)
# Initialize the AssetResolver singleton
AssetResolver(data_directory=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko)
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='periodically_query_icons_until_all_cached',
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.lock.release()
self.shutdown_event = gevent.event.Event()
def get_vault_normalized_balance(vault: MakerDAOVault) -> Balance:
"""Get the balance in the vault's collateral asset after deducting the generated debt"""
collateral_usd_price = Inquirer().find_usd_price(vault.collateral_asset)
normalized_usd_value = vault.collateral.usd_value - vault.debt.usd_value
return Balance(
amount=normalized_usd_value / collateral_usd_price,
usd_value=normalized_usd_value,
)
def modify_eth_account(
self,
account: ChecksumEthAddress,
append_or_remove: str,
) -> None:
"""Either appends or removes an ETH acccount.
Call with 'append' to add the account
Call with 'remove' remove the account
May raise:
- Input error if the given_account is not a valid ETH address
- BadFunctionCallOutput if a token is queried from a local chain
and the chain is not synced
- RemoteError if there is a problem with a query to an external
service such as Etherscan or cryptocompare
"""
eth_usd_price = Inquirer().find_usd_price(A_ETH)
remove_with_populated_balance = (
append_or_remove == 'remove' and len(self.balances.eth) != 0
)
# Query the balance of the account except for the case when it's removed
# and there is no other account in the balances
if append_or_remove == 'append' or remove_with_populated_balance:
amount = self.ethereum.get_eth_balance(account)
usd_value = amount * eth_usd_price
if append_or_remove == 'append':
self.accounts.eth.append(account)
self.balances.eth[account] = BalanceSheet(
assets=defaultdict(Balance, {A_ETH: Balance(amount, usd_value)}),
)
# Check if the new account has any staked eth2 deposits
self.account_for_staked_eth2_balances([account], at_addition=True)
elif append_or_remove == 'remove':
if account not in self.accounts.eth:
raise InputError('Tried to remove a non existing ETH account')
self.accounts.eth.remove(account)
balances = self.balances.eth.get(account, None)
if balances is not None:
for asset, balance in balances.assets.items():
self.totals.assets[asset] -= balance
if self.totals.assets[asset].amount <= ZERO:
self.totals.assets[asset] = Balance()
self.balances.eth.pop(account, None)
else:
raise AssertionError('Programmer error: Should be append or remove')
if len(self.balances.eth) == 0:
# If the last account was removed balance should be 0
self.totals.assets[A_ETH] = Balance()
elif append_or_remove == 'append':
self.totals.assets[A_ETH] += Balance(amount, usd_value)
self._query_ethereum_tokens(
action=AccountAction.APPEND,
given_accounts=[account],
)
def _query_lending_balances(
self,
balances: DefaultDict[Asset, Balance],
) -> DefaultDict[Asset, Balance]:
"""Queries binance lending balances and if any found adds them to `balances`
May raise:
- RemoteError
"""
data = self.api_query_dict('sapi', 'lending/union/account')
positions = data.get('positionAmountVos', None)
if positions is None:
raise RemoteError(
f'Could not find key positionAmountVos in lending account data '
f'{data} returned by {self.name}.', )
for entry in positions:
try:
amount = deserialize_asset_amount(entry['amount'])
if amount == ZERO:
continue
asset = asset_from_binance(entry['asset'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported {self.name} asset {e.asset_name}. '
f'Ignoring its lending balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown {self.name} asset {e.asset_name}. '
f'Ignoring its lending balance query.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Error at deserializing {self.name} asset. {msg}. '
f'Ignoring its lending balance query.', )
continue
try:
usd_price = Inquirer().find_usd_price(asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing {self.name} balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
balances[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
return balances
def query_balances(self) -> ExchangeQueryBalances:
try:
resp = self.api_query('balances')
except RemoteError as e:
msg = ('Bittrex API request failed. Could not reach bittrex due '
'to {}'.format(e))
log.error(msg)
return None, msg
returned_balances: Dict[Asset, Balance] = {}
for entry in resp:
try:
asset = asset_from_bittrex(entry['currencySymbol'])
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported bittrex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown bittrex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found bittrex asset with non-string type {type(entry["Currency"])}'
f' Ignoring its balance query.', )
continue
if entry['currencySymbol'] == 'BTXCRD':
# skip BTXCRD balance, since it's bittrex internal and we can't query usd price
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing bittrex balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
amount = FVal(entry['total'])
usd_value = amount * usd_price
returned_balances[asset] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'bittrex balance query result',
sensitive_log=True,
currency=asset,
amount=amount,
usd_value=usd_value,
)
return returned_balances, ''
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
This runs during backend initialization so it should be as light as possible.
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
self.data_dir.mkdir(parents=True, exist_ok=True)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.main_loop_spawned = False
self.args = args
self.api_task_greenlets: List[gevent.Greenlet] = []
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
# Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver
GlobalDBHandler(data_dir=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir,
coingecko=self.coingecko)
self.assets_updater = AssetsUpdater(self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.task_manager: Optional[TaskManager] = None
self.shutdown_event = gevent.event.Event()
def __init__(self, args: argparse.Namespace) -> None:
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise AssertionError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
self.data_dir = args.data_dir
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def query_balances(self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
resp = self.api_query_dict('returnCompleteBalances', {"account": "all"})
except (RemoteError, PoloniexError) as e:
msg = (
'Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
balances = dict()
for poloniex_asset, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if (available != FVal(0) or on_orders != FVal(0)):
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.',
)
continue
except DeserializationError:
log.error(
f'Unexpected poloniex asset type. Expected string '
f' but got {type(poloniex_asset)}',
)
self.msg_aggregator.add_error(
'Found poloniex asset entry with non-string type. '
' Ignoring its balance query.',
)
entry = {}
entry['amount'] = available + on_orders
usd_price = Inquirer().find_usd_price(asset=asset)
usd_value = entry['amount'] * usd_price
entry['usd_value'] = usd_value
balances[asset] = entry
log.debug(
'Poloniex balance query',
sensitive_log=True,
currency=asset,
amount=entry['amount'],
usd_value=usd_value,
)
return balances, ''
def query_ethereum_tokens(
self,
tokens: List[EthereumToken],
) -> None:
"""Queries the ethereum token balances and populates the state
May raise:
- RemoteError if an external service such as Etherscan or cryptocompare
is queried and there is a problem with its query.
- EthSyncError if querying the token balances through a provided ethereum
client and the chain is not synced
"""
try:
self._query_ethereum_tokens_alethio(action=AccountAction.QUERY)
except RemoteError as e:
log.debug(
f'Alethio accounts token balances query failed: {str(e)}. '
f'Switching to etherscan/own node query.', )
self._query_ethereum_tokens_normal(tokens,
action=AccountAction.QUERY)
# If we have anything in DSR also count it towards total blockchain balances
eth_balances = self.balances.eth
if self.makerdao:
additional_total_dai = FVal(0)
try:
usd_price = Inquirer().find_usd_price(A_DAI)
except RemoteError:
# Let's try to continue with a usd/dai price of 1 if error
usd_price = Price(FVal('1'))
current_dsr_report = self.makerdao.get_current_dsr()
for dsr_account, dai_value in current_dsr_report.balances.items():
if dai_value == ZERO:
continue
usd_value = dai_value * usd_price
old_balance = eth_balances[dsr_account].asset_balances.get(
A_DAI,
Balance(amount=ZERO, usd_value=ZERO),
)
eth_balances[dsr_account].asset_balances[A_DAI] = Balance(
amount=old_balance.amount + dai_value,
usd_value=old_balance.usd_value + usd_value,
)
eth_balances[dsr_account].increase_total_usd_value(usd_value)
additional_total_dai += dai_value
old_total = self.totals.get(A_DAI,
Balance(amount=ZERO, usd_value=ZERO))
new_total_amount = old_total.amount + additional_total_dai
if new_total_amount != ZERO:
self.totals[A_DAI] = Balance(
amount=new_total_amount,
usd_value=new_total_amount * usd_price,
)
def query_balances(self) -> ExchangeQueryBalances:
try:
accounts, _ = self._api_query('accounts')
except (CoinbaseProPermissionError, RemoteError) as e:
msg = f'Coinbase Pro API request failed. {str(e)}'
log.error(msg)
return None, msg
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
for account in accounts:
try:
amount = deserialize_asset_amount(account['balance'])
# ignore empty balances. Coinbase returns zero balances for everything
# a user does not own
if amount == ZERO:
continue
asset = asset_from_coinbasepro(account['currency'])
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing coinbasepro balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
assets_balance[asset] += Balance(
amount=amount,
usd_value=amount * usd_price,
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro balance result with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found coinbase pro balance result with unsupported asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a coinbase pro account balance. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a coinbase pro account balance',
account_balance=account,
error=msg,
)
continue
return dict(assets_balance), ''
def query_balances(self) -> ExchangeQueryBalances:
try:
kraken_balances = self.api_query('Balance', req={})
except RemoteError as e:
if "Missing key: 'result'" in str(e):
# handle https://github.com/rotki/rotki/issues/946
kraken_balances = {}
else:
msg = ('Kraken API request failed. Could not reach kraken due '
'to {}'.format(e))
log.error(msg)
return None, msg
assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance)
for kraken_name, amount_ in kraken_balances.items():
amount = FVal(amount_)
if amount == ZERO:
continue
try:
our_asset = asset_from_kraken(kraken_name)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported/unknown kraken asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
self.msg_aggregator.add_error(
f'Found kraken asset with non-string type {type(kraken_name)}. '
f' Ignoring its balance query.', )
continue
balance = Balance(amount=amount)
if our_asset.identifier != 'KFEE':
# There is no price value for KFEE. TODO: Shouldn't we then just skip the balance?
try:
usd_price = Inquirer().find_usd_price(our_asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing kraken balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
balance.usd_value = balance.amount * usd_price
assets_balance[our_asset] += balance
log.debug(
'kraken balance query result',
sensitive_log=True,
currency=our_asset,
amount=balance.amount,
usd_value=balance.usd_value,
)
return dict(assets_balance), ''
def query_balances(self) -> Tuple[Optional[Dict[Asset, Balance]], str]:
"""Return the account balances on Bistamp
The balance endpoint returns a dict where the keys (str) are related to
assets and the values (str) amounts. The keys that end with `_balance`
contain the exact amount of an asset the account is holding (available
amount + orders amount, per asset).
"""
response = self._api_query('balance')
if response.status_code != HTTPStatus.OK:
result, msg = self._process_unsuccessful_response(
response=response,
case='balances',
)
return result, msg
try:
response_dict = rlk_jsonloads_dict(response.text)
except JSONDecodeError as e:
msg = f'Bitstamp returned invalid JSON response: {response.text}.'
log.error(msg)
raise RemoteError(msg) from e
asset_balance: Dict[Asset, Balance] = {}
for entry, amount in response_dict.items():
amount = FVal(amount)
if not entry.endswith('_balance') or amount == ZERO:
continue
symbol = entry.split('_')[0] # If no `_`, defaults to entry
try:
asset = Asset(symbol)
except (UnknownAsset, UnsupportedAsset) as e:
log.error(str(e))
asset_tag = 'unknown' if isinstance(
e, UnknownAsset) else 'unsupported'
self.msg_aggregator.add_warning(
f'Found {asset_tag} Bistamp asset {e.asset_name}. Ignoring its balance query.',
)
continue
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
log.error(str(e))
self.msg_aggregator.add_error(
f'Error processing Bitstamp balance result due to inability to '
f'query USD price: {str(e)}. Skipping balance entry.', )
continue
asset_balance[asset] = Balance(
amount=amount,
usd_value=amount * usd_price,
)
return asset_balance, ''
def __init__(self, data_directory='.'):
self.data_directory = Path(data_directory)
self.config = yaml.load(open(self.data_directory / 'buchfink.yaml',
'r'),
Loader=yaml.SafeLoader)
self.reports_directory = self.data_directory / "reports"
self.trades_directory = self.data_directory / "trades"
self.cache_directory = self.data_directory / "cache"
self.reports_directory.mkdir(exist_ok=True)
self.trades_directory.mkdir(exist_ok=True)
self.cache_directory.mkdir(exist_ok=True)
(self.cache_directory / 'cryptocompare').mkdir(exist_ok=True)
(self.cache_directory / 'history').mkdir(exist_ok=True)
(self.cache_directory / 'inquirer').mkdir(exist_ok=True)
self.cryptocompare = Cryptocompare(
self.cache_directory / 'cryptocompare', self)
self.historian = PriceHistorian(self.cache_directory / 'history',
'01/01/2014', self.cryptocompare)
self.inquirer = Inquirer(self.cache_directory / 'inquirer',
self.cryptocompare)
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
# Initialize blockchain querying modules
self.etherscan = Etherscan(database=self,
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.alethio = Alethio(
database=self,
msg_aggregator=self.msg_aggregator,
all_eth_tokens=self.all_eth_tokens,
)
self.ethereum_manager = EthereumManager(
ethrpc_endpoint=self.get_eth_rpc_endpoint(),
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
)
#self.chain_manager = ChainManager(
# blockchain_accounts=[],
# owned_eth_tokens=[],
# ethereum_manager=self.ethereum_manager,
# msg_aggregator=self.msg_aggregator,
# alethio=alethio,
# greenlet_manager=self.greenlet_manager,
# premium=False,
# eth_modules=ethereum_modules,
#)
self.ethereum_analyzer = EthereumAnalyzer(
ethereum_manager=self.ethereum_manager,
database=self,
)
