def test_coverage_of_kraken_balances(kraken): # Since 05/08/2019 Kraken removed all delisted assets from their public API # query except for BSV. No idea why or why this incosistency. got_assets = set(kraken.api_query('Assets').keys()) expected_assets = (set(KRAKEN_TO_WORLD.keys()) - set(KRAKEN_DELISTED)).union({'BSV'}) # Ignore the staking assets from the got assets got_assets.remove('XTZ.S') got_assets.remove('EUR.M') got_assets.remove('USD.M') got_assets.remove('XBT.M') diff = expected_assets.symmetric_difference(got_assets) if len(diff) != 0: test_warnings.warn(UserWarning( f"Our known assets don't match kraken's assets. Difference: {diff}", )) else: # Make sure all assets are covered by our from and to functions for kraken_asset in got_assets: asset = asset_from_kraken(kraken_asset) assert asset.to_kraken() == kraken_asset # also check that staked assets are properly processed assert asset_from_kraken('XTZ.S') == Asset('XTZ') assert asset_from_kraken('EUR.M') == Asset('EUR')
def kraken_to_world_pair(pair: str) -> TradePair: """Turns a pair from kraken to our pair type Can throw: - UknownAsset if one of the assets of the pair are not known - UnprocessableKrakenPair if the pair can't be processed and split into its base/quote assets """ # handle dark pool pairs if pair[-2:] == '.d': pair = pair[:-2] if pair[0:3] in KRAKEN_ASSETS: base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:4] in KRAKEN_ASSETS: base_asset_str = pair[0:4] quote_asset_str = pair[4:] else: raise UnprocessableTradePair(pair) base_asset = asset_from_kraken(base_asset_str) quote_asset = asset_from_kraken(quote_asset_str) return trade_pair_from_assets(base_asset, quote_asset)
def test_coverage_of_kraken_balances(kraken): got_assets = set(kraken.api_query('Assets').keys()) expected_assets = (set(KRAKEN_TO_WORLD.keys()) - set(KRAKEN_DELISTED)) # Ignore the staking assets from the got assets got_assets.remove('XTZ.S') got_assets.remove('DOT.S') got_assets.remove('ATOM.S') got_assets.remove('EUR.M') got_assets.remove('USD.M') got_assets.remove('XBT.M') got_assets.remove('KSM.S') got_assets.remove('ETH2.S') got_assets.remove('EUR.HOLD') got_assets.remove('FLOW.S') got_assets.remove('FLOWH.S') # Ignore the following assets as well got_assets.remove('FLOW') got_assets.remove('FLOWH') diff = expected_assets.symmetric_difference(got_assets) if len(diff) != 0: test_warnings.warn(UserWarning( f"Our known assets don't match kraken's assets. Difference: {diff}", )) else: # Make sure all assets are covered by our from and to functions for kraken_asset in got_assets: asset = asset_from_kraken(kraken_asset) assert asset.to_kraken() == kraken_asset # also check that staked assets are properly processed assert asset_from_kraken('XTZ.S') == Asset('XTZ') assert asset_from_kraken('EUR.M') == Asset('EUR')
def query_balances(self) -> ExchangeQueryBalances: try: kraken_balances = self.api_query('Balance', req={}) except RemoteError as e: if "Missing key: 'result'" in str(e): # handle https://github.com/rotki/rotki/issues/946 kraken_balances = {} else: msg = ( 'Kraken API request failed. Could not reach kraken due ' 'to {}'.format(e) ) log.error(msg) return None, msg assets_balance: DefaultDict[Asset, Balance] = defaultdict(Balance) for kraken_name, amount_ in kraken_balances.items(): amount = FVal(amount_) if amount == ZERO: continue try: our_asset = asset_from_kraken(kraken_name) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unsupported/unknown kraken asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except DeserializationError: self.msg_aggregator.add_error( f'Found kraken asset with non-string type {type(kraken_name)}. ' f' Ignoring its balance query.', ) continue balance = Balance(amount=amount) if our_asset.identifier != 'KFEE': # There is no price value for KFEE. TODO: Shouldn't we then just skip the balance? try: usd_price = Inquirer().find_usd_price(our_asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing kraken balance entry due to inability to ' f'query USD price: {str(e)}. Skipping balance entry', ) continue balance.usd_value = balance.amount * usd_price assets_balance[our_asset] += balance log.debug( 'kraken balance query result', sensitive_log=True, currency=our_asset, amount=balance.amount, usd_value=balance.usd_value, ) return dict(assets_balance), ''
def query_balances(self) -> Tuple[Optional[dict], str]: try: self.first_connection() old_balances = self.query_private('Balance', req={}) except RemoteError as e: msg = ('Kraken API request failed. Could not reach kraken due ' 'to {}'.format(e)) log.error(msg) return None, msg balances = dict() for k, v in old_balances.items(): v = FVal(v) if v == FVal(0): continue our_asset = asset_from_kraken(k) entry = {} entry['amount'] = v if our_asset in self.usdprice: entry['usd_value'] = v * self.usdprice[our_asset] else: entry['usd_value'] = v * self.find_fiat_price(k) balances[our_asset] = entry log.debug( 'kraken balance query result', sensitive_log=True, currency=our_asset, amount=entry['amount'], usd_value=entry['usd_value'], ) return balances, ''
def kraken_to_world_pair(pair: str) -> Tuple[Asset, Asset]: """Turns a pair from kraken to our base/quote asset tuple Can throw: - UknownAsset if one of the assets of the pair are not known - DeserializationError if one of the assets is not a sting - UnprocessableTradePair if the pair can't be processed and split into its base/quote assets """ # handle dark pool pairs if pair[-2:] == '.d': pair = pair[:-2] if len(pair) == 6 and pair[0:3] in ('EUR', 'USD', 'AUD'): # This is for the FIAT to FIAT pairs that kraken introduced base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair == 'ETHDAI': return A_ETH, A_DAI elif pair == 'ETH2.SETH': return A_ETH2, A_ETH elif pair[0:2] in KRAKEN_TO_WORLD: base_asset_str = pair[0:2] quote_asset_str = pair[2:] elif pair[0:3] in KRAKEN_TO_WORLD: base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:3] in ('XBT', 'ETH', 'XDG', 'LTC', 'XRP'): # Some assets can have the 'X' prefix omitted for some pairs base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:4] in KRAKEN_TO_WORLD: base_asset_str = pair[0:4] quote_asset_str = pair[4:] elif pair[0:5] in KRAKEN_TO_WORLD: base_asset_str = pair[0:5] quote_asset_str = pair[5:] elif pair[0:6] in KRAKEN_TO_WORLD: base_asset_str = pair[0:6] quote_asset_str = pair[6:] else: raise UnprocessableTradePair(pair) base_asset = asset_from_kraken(base_asset_str) quote_asset = asset_from_kraken(quote_asset_str) return base_asset, quote_asset
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List: with self.lock: cache = self.check_trades_cache( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: movements.append( AssetMovement( exchange='kraken', category=movement['type'], # Kraken timestamps have floating point timestamp=convert_to_int(movement['time'], accept_only_exact=False), asset=asset_from_kraken(movement['asset']), amount=FVal(movement['amount']), fee=FVal(movement['fee']), )) return movements
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend(self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: asset = asset_from_kraken(movement['asset']) movements.append(AssetMovement( location=Location.KRAKEN, category=deserialize_asset_movement_category(movement['type']), timestamp=deserialize_timestamp_from_kraken(movement['time']), asset=asset, amount=deserialize_asset_amount(movement['amount']), fee_asset=asset, fee=deserialize_fee(movement['fee']), link=str(movement['refid']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a kraken deposit/withdrawals. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a kraken deposit/withdrawal.', raw_asset_movement=movement, error=msg, ) continue return movements
def kraken_to_world_pair(pair: str) -> TradePair: # handle dark pool pairs if pair[-2:] == '.d': pair = pair[:-2] if pair[0:3] in KRAKEN_ASSETS: base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:4] in KRAKEN_ASSETS: base_asset_str = pair[0:4] quote_asset_str = pair[4:] else: raise ValueError(f'Could not process kraken trade pair {pair}') base_asset = asset_from_kraken(base_asset_str) quote_asset = asset_from_kraken(quote_asset_str) return trade_pair_from_assets(base_asset, quote_asset)
def test_coverage_of_kraken_balances(kraken): all_assets = set(kraken.query_public('Assets').keys()) diff = set(KRAKEN_ASSETS).symmetric_difference(all_assets) assert len(diff) == 0, ( f"Our known assets don't match kraken's assets. Difference: {diff}") # Make sure all assets are covered by our from and to functions for kraken_asset in all_assets: asset = asset_from_kraken(kraken_asset) assert asset.to_kraken() == kraken_asset
def query_balances(self) -> Tuple[Optional[dict], str]: try: old_balances = self.api_query('Balance', req={}) except RemoteError as e: if "Missing key: 'result'" in str(e): # handle https://github.com/rotki/rotki/issues/946 old_balances = {} else: msg = ('Kraken API request failed. Could not reach kraken due ' 'to {}'.format(e)) log.error(msg) return None, msg balances = {} for k, v in old_balances.items(): v = FVal(v) if v == FVal(0): continue try: our_asset = asset_from_kraken(k) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unsupported/unknown kraken asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except DeserializationError: self.msg_aggregator.add_error( f'Found kraken asset with non-string type {type(k)}. ' f' Ignoring its balance query.', ) continue entry = {} entry['amount'] = v if k == 'KFEE': # There is no price value for KFEE. TODO: Shouldn't we then just skip the balance? entry['usd_value'] = ZERO else: try: usd_price = Inquirer().find_usd_price(our_asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing kraken balance entry due to inability to ' f'query USD price: {str(e)}. Skipping balance entry', ) continue entry['usd_value'] = FVal(v * usd_price) balances[our_asset] = entry log.debug( 'kraken balance query result', sensitive_log=True, currency=our_asset, amount=entry['amount'], usd_value=entry['usd_value'], ) return balances, ''
def kraken_to_world_pair(pair: str) -> TradePair: """Turns a pair from kraken to our pair type Can throw: - UknownAsset if one of the assets of the pair are not known - UnprocessableKrakenPair if the pair can't be processed and split into its base/quote assets """ # handle dark pool pairs if pair[-2:] == '.d': pair = pair[:-2] if len(pair) == 6 and pair[0:3] in ('EUR', 'USD', 'AUD'): # This is for the FIAT to FIAT pairs that kraken introduced base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair == 'ETHDAI': return trade_pair_from_assets(base=Asset('ETH'), quote=EthereumToken('DAI')) elif pair[0:2] in KRAKEN_TO_WORLD: base_asset_str = pair[0:2] quote_asset_str = pair[2:] elif pair[0:3] in KRAKEN_TO_WORLD: base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:3] in ('XBT', 'ETH', 'XDG', 'LTC', 'XRP'): # Some assets can have the 'X' prefix omitted for some pairs base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:4] in KRAKEN_TO_WORLD: base_asset_str = pair[0:4] quote_asset_str = pair[4:] elif pair[0:5] in KRAKEN_TO_WORLD: base_asset_str = pair[0:5] quote_asset_str = pair[5:] else: raise UnprocessableTradePair(pair) base_asset = asset_from_kraken(base_asset_str) quote_asset = asset_from_kraken(quote_asset_str) return trade_pair_from_assets(base_asset, quote_asset)
def test_coverage_of_kraken_balances(kraken): # Since 05/08/2019 Kraken removed all delisted assets from their public API # query except for BSV. No idea why or why this incosistency. got_assets = set(kraken.query_public('Assets').keys()) expected_assets = (set(KRAKEN_ASSETS) - set(KRAKEN_DELISTED)).union(set(['BSV'])) diff = expected_assets.symmetric_difference(got_assets) assert len(diff) == 0, ( f"Our known assets don't match kraken's assets. Difference: {diff}" ) # Make sure all assets are covered by our from and to functions for kraken_asset in got_assets: asset = asset_from_kraken(kraken_asset) assert asset.to_kraken() == kraken_asset
def kraken_to_world_pair(pair: str) -> TradePair: """Turns a pair from kraken to our pair type Can throw: - UknownAsset if one of the assets of the pair are not known - UnprocessableKrakenPair if the pair can't be processed and split into its base/quote assets """ # handle dark pool pairs if pair[-2:] == '.d': pair = pair[:-2] if pair == 'ETHDAI': return trade_pair_from_assets(base=A_ETH, quote=A_DAI) elif pair[0:2] in KRAKEN_TO_WORLD: base_asset_str = pair[0:2] quote_asset_str = pair[2:] elif pair[0:3] in KRAKEN_TO_WORLD: base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:3] in ('XBT', 'ETH', 'XDG'): # Some assets can have the 'X' prefix omitted for some pairs base_asset_str = pair[0:3] quote_asset_str = pair[3:] elif pair[0:4] in KRAKEN_TO_WORLD: base_asset_str = pair[0:4] quote_asset_str = pair[4:] elif pair[0:5] in KRAKEN_TO_WORLD: base_asset_str = pair[0:5] quote_asset_str = pair[5:] else: raise UnprocessableTradePair(pair) base_asset = asset_from_kraken(base_asset_str) quote_asset = asset_from_kraken(quote_asset_str) return trade_pair_from_assets(base_asset, quote_asset)
def find_fiat_price(self, kraken_asset: str) -> FVal: """Find USD/EUR price of asset. The asset should be in the kraken style. e.g.: XICN. Save both prices in the kraken object and then return the USD price. """ if kraken_asset == 'KFEE': # Kraken fees have no value return FVal(0) if kraken_asset == 'XXBT': return self.usdprice['BTC'] if kraken_asset == 'USDT': price = FVal(self.ticker['USDTZUSD']['c'][0]) self.usdprice['USDT'] = price return price if kraken_asset == 'BSV': # BSV has been delisted by kraken at 29/04/19 # https://blog.kraken.com/post/2274/kraken-is-delisting-bsv/ # Until May 31st there can be BSV in Kraken (even with 0 balance) # so keep this until then to get the price return query_cryptocompare_for_fiat_price(A_BSV) # TODO: This is pretty ugly. Find a better way to check out kraken pairs # without this ugliness. pair = kraken_asset + 'XXBT' pair2 = kraken_asset + 'XBT' pair3 = 'XXBT' + kraken_asset inverse = False if pair2 in self.tradeable_pairs: pair = pair2 elif pair3 in self.tradeable_pairs: pair = pair3 # here XXBT is the base asset so inverse inverse = True if pair not in self.tradeable_pairs: raise ValueError( 'Could not find a BTC tradeable pair in kraken for "{}"'. format(kraken_asset), ) btc_price = FVal(self.ticker[pair]['c'][0]) if inverse: btc_price = FVal('1.0') / btc_price our_asset = asset_from_kraken(kraken_asset) with self.lock: self.usdprice[our_asset] = btc_price * self.usdprice['BTC'] self.eurprice[our_asset] = btc_price * self.eurprice['BTC'] return self.usdprice[our_asset]
def query_balances(self) -> Tuple[Optional[dict], str]: try: old_balances = self.query_private('Balance', req={}) except RemoteError as e: msg = ( 'Kraken API request failed. Could not reach kraken due ' 'to {}'.format(e) ) log.error(msg) return None, msg balances = dict() for k, v in old_balances.items(): v = FVal(v) if v == FVal(0): continue try: our_asset = asset_from_kraken(k) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unsupported/unknown kraken asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except DeserializationError: self.msg_aggregator.add_error( f'Found kraken asset with non-string type {type(k)}. ' f' Ignoring its balance query.', ) continue entry = {} entry['amount'] = v usd_price = Inquirer().find_usd_price(our_asset) entry['usd_value'] = FVal(v * usd_price) balances[our_asset] = entry log.debug( 'kraken balance query result', sensitive_log=True, currency=our_asset, amount=entry['amount'], usd_value=entry['usd_value'], ) return balances, ''
def history_event_from_kraken( events: List[Dict[str, Any]], name: str, msg_aggregator: MessagesAggregator, ) -> Tuple[List[HistoryBaseEntry], bool]: """ This function gets raw data from kraken and creates a list of related history events to be used in the app. It returns a list of events and a boolean in the case that an unknown type is found. """ group_events = [] found_unknown_event = False current_fee_index = len(events) for idx, raw_event in enumerate(events): try: timestamp = TimestampMS((deserialize_fval( value=raw_event['time'], name='time', location='kraken ledger processing', ) * 1000).to_int(exact=False)) identifier = raw_event['refid'] event_type = kraken_ledger_entry_type_to_ours(raw_event['type']) asset = asset_from_kraken(raw_event['asset']) event_subtype = HistoryEventSubType.NONE notes = None raw_amount = deserialize_asset_amount(raw_event['amount']) # If we don't know how to handle an event atm or we find an unsupported # event type the logic will be to store it as unknown and if in the future # we need some information from it we can take actions to process them if event_type == HistoryEventType.TRANSFER: if raw_event['subtype'] == 'spottostaking': event_type = HistoryEventType.STAKING event_subtype = HistoryEventSubType.DEPOSIT_ASSET elif raw_event['subtype'] == 'stakingfromspot': event_type = HistoryEventType.STAKING event_subtype = HistoryEventSubType.RECEIVE_WRAPPED elif raw_event['subtype'] == 'stakingtospot': event_type = HistoryEventType.STAKING event_subtype = HistoryEventSubType.REMOVE_ASSET elif raw_event['subtype'] == 'spotfromstaking': event_type = HistoryEventType.STAKING event_subtype = HistoryEventSubType.RETURN_WRAPPED elif event_type == HistoryEventType.ADJUSTMENT: if raw_amount < ZERO: event_subtype = HistoryEventSubType.SPEND else: event_subtype = HistoryEventSubType.RECEIVE elif event_type == HistoryEventType.STAKING: event_subtype = HistoryEventSubType.REWARD elif event_type == HistoryEventType.INFORMATIONAL: found_unknown_event = True notes = raw_event['type'] log.warning( f'Encountered kraken historic event type we do not process. {raw_event}', ) fee_amount = deserialize_asset_amount(raw_event['fee']) # Make sure to not generate an event for KFEES that is not of type FEE if asset != A_KFEE: group_events.append( HistoryBaseEntry( event_identifier=identifier, sequence_index=idx, timestamp=timestamp, location=Location.KRAKEN, location_label=name, asset=asset, balance=Balance( amount=raw_amount, usd_value=ZERO, ), notes=notes, event_type=event_type, event_subtype=event_subtype, )) if fee_amount != ZERO: group_events.append( HistoryBaseEntry( event_identifier=identifier, sequence_index=current_fee_index, timestamp=timestamp, location=Location.KRAKEN, location_label=name, asset=asset, balance=Balance( amount=fee_amount, usd_value=ZERO, ), notes=notes, event_type=event_type, event_subtype=HistoryEventSubType.FEE, )) # Increase the fee index to not have duplicates in the case of having a normal # fee and KFEE current_fee_index += 1 except (DeserializationError, KeyError, UnknownAsset) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Keyrror {msg}' msg_aggregator.add_error( f'Failed to read ledger event from kraken {raw_event} due to {msg}', ) return [], False return group_events, found_unknown_event
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_list( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) result: List[Any] if cache is not None: result = cache else: result = self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='deposit'), ) result.extend( self.query_until_finished( endpoint='Ledgers', keyname='ledger', start_ts=start_ts, end_ts=end_ts, extra_dict=dict(type='withdrawal'), )) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) log.debug('Kraken deposit/withdrawals query result', num_results=len(result)) movements = list() for movement in result: try: movements.append( AssetMovement( exchange='kraken', category=movement['type'], # Kraken timestamps have floating point timestamp=Timestamp( convert_to_int(movement['time'], accept_only_exact=False)), asset=asset_from_kraken(movement['asset']), amount=FVal(movement['amount']), fee=Fee(FVal(movement['fee'])), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown kraken asset {e.asset_name}. ' f'Ignoring its deposit/withdrawals query.', ) continue return movements