def _get_cached_forex_data( date: str, from_currency: Asset, to_currency: Asset, ) -> Optional[Price]: instance = Inquirer() rate = None if date in instance._cached_forex_data: if from_currency in instance._cached_forex_data[date]: rate = instance._cached_forex_data[date][from_currency].get( to_currency) if rate: log.debug( 'Got cached forex rate', from_currency=from_currency.identifier, to_currency=to_currency.identifier, rate=rate, ) try: rate = deserialize_price(rate) except DeserializationError as e: log.error( f'Could not read cached forex entry due to {str(e)}' ) return rate
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]: """Turns an FTX transaction into a rotki Trade. May raise: - UnknownAsset due to Asset instantiation - DeserializationError due to unexpected format of dict entries - KeyError due to dict entries missing an expected key """ # In the case of perpetuals and futures this fields can be None if raw_trade.get('baseCurrency', None) is None: return None if raw_trade.get('quoteCurrency', None) is None: return None timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601', 'FTX') trade_type = deserialize_trade_type(raw_trade['side']) base_asset = asset_from_ftx(raw_trade['baseCurrency']) quote_asset = asset_from_ftx(raw_trade['quoteCurrency']) amount = deserialize_asset_amount(raw_trade['size']) rate = deserialize_price(raw_trade['price']) fee_currency = asset_from_ftx(raw_trade['feeCurrency']) fee = deserialize_fee(raw_trade['fee']) return Trade( timestamp=timestamp, location=Location.FTX, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(raw_trade['id']), )
def deserialize_from_db( cls, event_tuple: BalancerEventDBTuple, ) -> 'BalancerEvent': """May raise DeserializationError Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address_token 6 - lp_amount 7 - usd_value 8 - amount0 9 - amount1 10 - amount2 11 - amount3 12 - amount4 13 - amount5 14 - amount6 15 - amount7 """ event_tuple_type = event_tuple[4] try: event_type = getattr(BalancerBPTEventType, event_tuple_type.upper()) except AttributeError as e: raise DeserializationError( f'Unexpected event type: {event_tuple_type}.') from e pool_address_token = EthereumToken.from_identifier( event_tuple[5], form_with_incomplete_data= True, # since some may not have decimals input correctly ) if pool_address_token is None: raise DeserializationError( f'Balancer event pool token: {event_tuple[5]} not found in the DB.', ) amounts: List[AssetAmount] = [ deserialize_asset_amount(item) for item in event_tuple[8:16] if item is not None ] return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address_token=pool_address_token, lp_balance=Balance( amount=deserialize_asset_amount(event_tuple[6]), usd_value=deserialize_price(event_tuple[7]), ), amounts=amounts, )
def deserialize_trade(data: Dict[str, Any]) -> Trade: """ Takes a dict trade representation of our common trade format and serializes it into the Trade object May raise: - UnknownAsset: If the base, quote, fee asset string is not a known asset - DeserializationError: If any of the trade dict entries is not as expected """ rate = deserialize_price(data['rate']) amount = deserialize_asset_amount(data['amount']) trade_type = TradeType.deserialize(data['trade_type']) location = Location.deserialize(data['location']) trade_link = '' if 'link' in data: trade_link = data['link'] trade_notes = '' if 'notes' in data: trade_notes = data['notes'] return Trade( timestamp=data['timestamp'], location=location, base_asset=Asset(data['base_asset']), quote_asset=Asset(data['quote_asset']), trade_type=trade_type, amount=amount, rate=rate, fee=deserialize_fee(data['fee']), fee_currency=Asset(data['fee_currency']), link=trade_link, notes=trade_notes, )
def get_bisq_market_price(asset: Asset) -> Price: """ Get price for pair at bisq marketplace. Price is returned against BTC. Can raise: - RemoteError: If the market doesn't exists or request fails - DeserializationError: If the data returned is not a valid price """ url = PRICE_API_URL.format(symbol=asset.symbol) try: response = requests.get(url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError( f'bisq.markets request {url} failed due to {str(e)}') from e try: data = response.json() except json.decoder.JSONDecodeError as e: raise RemoteError( f'Failed to read json response from bisq.markets. {response.text}. {str(e)}', ) from e if 'error' in data: raise RemoteError( f'Request data from bisq.markets {url} is not valid {data["error"]}' ) try: price = data['last'] except KeyError as e: raise DeserializationError( f'Response from bisq.markets didnt contain expected key "last". {data}', ) from e return deserialize_price(price)
def _deserialize_trade(self, raw_result: List[Any]) -> Trade: """Process a trade result from Bitfinex and deserialize it. The base and quote assets are instantiated using the `fee_currency_symbol` (from raw_result[10]) over the pair (from raw_result[1]). Known pairs format: 'tETHUST', 'tETH:UST'. Can raise DeserializationError. Schema reference in: https://docs.bitfinex.com/reference#rest-auth-trades """ amount = deserialize_asset_amount(raw_result[4]) trade_type = TradeType.BUY if amount >= ZERO else TradeType.SELL bfx_pair = self._process_bfx_pair(raw_result[1]) if bfx_pair not in self.pair_bfx_symbols_map: raise DeserializationError( f'Could not deserialize bitfinex trade pair {raw_result[1]}. ' f'Raw trade: {raw_result}', ) bfx_base_asset_symbol, bfx_quote_asset_symbol = self.pair_bfx_symbols_map[ bfx_pair] try: base_asset = asset_from_bitfinex( bitfinex_name=bfx_base_asset_symbol, currency_map=self.currency_map, ) quote_asset = asset_from_bitfinex( bitfinex_name=bfx_quote_asset_symbol, currency_map=self.currency_map, ) fee_asset = asset_from_bitfinex( bitfinex_name=raw_result[10], currency_map=self.currency_map, ) except (UnknownAsset, UnsupportedAsset) as e: asset_tag = 'Unknown' if isinstance( e, UnknownAsset) else 'Unsupported' raise DeserializationError( f'{asset_tag} {e.asset_name} found while processing trade pair due to: {str(e)}', ) from e trade = Trade( timestamp=Timestamp(int(raw_result[2] / 1000)), location=Location.BITFINEX, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=AssetAmount(abs(amount)), rate=deserialize_price(raw_result[5]), fee=Fee(abs(deserialize_fee(raw_result[9]))), fee_currency=fee_asset, link=str(raw_result[0]), notes='', ) return trade
def deserialize_token_day_data( raw_token_day_data: Dict[str, Any], ) -> Tuple[ChecksumEthAddress, Price]: """May raise DeserializationError""" try: token_address = raw_token_day_data['token']['id'] usd_price = deserialize_price(raw_token_day_data['priceUSD']) except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e token_address = deserialize_ethereum_address(token_address) return token_address, usd_price
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade: """Turn a bittrex trade returned from bittrex trade history to our common trade history format As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible that some keys don't exist in a trade. The required fields are here: https://bittrex.github.io/api/v3#definition-Order May raise: - UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world() - DeserializationError due to unexpected format of dict entries - KeyError due to dict entries missing an expected entry """ amount = deserialize_asset_amount(bittrex_trade['fillQuantity']) timestamp = deserialize_timestamp_from_date( date=bittrex_trade['closedAt'], # we only check closed orders formatstr='iso8601', location='bittrex', ) if 'limit' in bittrex_trade: rate = deserialize_price(bittrex_trade['limit']) else: rate = Price( deserialize_asset_amount(bittrex_trade['proceeds']) / deserialize_asset_amount(bittrex_trade['fillQuantity']), ) order_type = deserialize_trade_type(bittrex_trade['direction']) fee = deserialize_fee(bittrex_trade['commission']) base_asset, quote_asset = bittrex_pair_to_world(bittrex_trade['marketSymbol']) log.debug( 'Processing bittrex Trade', sensitive_log=True, amount=amount, rate=rate, order_type=order_type, fee=fee, bittrex_pair=bittrex_trade['marketSymbol'], base_asset=base_asset, quote_asset=quote_asset, ) return Trade( timestamp=timestamp, location=Location.BITTREX, base_asset=base_asset, quote_asset=quote_asset, trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=quote_asset, link=str(bittrex_trade['id']), )
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - token0_identifier 7 - token1_identifier 8 - amount0 9 - amount1 10 - usd_price 11 - lp_amount """ db_event_type = event_tuple[4] if db_event_type not in {str(event_type) for event_type in EventType}: raise DeserializationError( f'Failed to deserialize event type. Unknown event: {db_event_type}.', ) if db_event_type == str(EventType.MINT): event_type = EventType.MINT elif db_event_type == str(EventType.BURN): event_type = EventType.BURN else: raise ValueError(f'Unexpected event type case: {db_event_type}.') token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6]) token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[8]), amount1=deserialize_asset_amount(event_tuple[9]), usd_price=deserialize_price(event_tuple[10]), lp_amount=deserialize_asset_amount(event_tuple[11]), )
def deserialize_from_db( cls, event_tuple: BalancerEventDBTuple, ) -> 'BalancerEvent': """May raise DeserializationError Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - lp_amount 7 - usd_value 8 - amount0 9 - amount1 10 - amount2 11 - amount3 12 - amount4 13 - amount5 14 - amount6 15 - amount7 """ event_tuple_type = event_tuple[4] try: event_type = getattr(BalancerBPTEventType, event_tuple_type.upper()) except AttributeError as e: raise DeserializationError( f'Unexpected event type: {event_tuple_type}.') from e amounts: List[AssetAmount] = [ deserialize_asset_amount(item) for item in event_tuple[8:16] if item is not None ] return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), lp_balance=Balance( amount=deserialize_asset_amount(event_tuple[6]), usd_value=deserialize_price(event_tuple[7]), ), amounts=amounts, )
def _deserialize_transaction(grant_id: int, rawtx: Dict[str, Any]) -> LedgerAction: """May raise: - DeserializationError - KeyError - UnknownAsset """ timestamp = deserialize_timestamp_from_date( date=rawtx['timestamp'], formatstr='%Y-%m-%dT%H:%M:%S', location='Gitcoin API', skip_milliseconds=True, ) asset = get_gitcoin_asset(symbol=rawtx['asset'], token_address=rawtx['token_address']) raw_amount = deserialize_int_from_str(symbol=rawtx['amount'], location='gitcoin api') amount = asset_normalized_value(raw_amount, asset) if amount == ZERO: raise ZeroGitcoinAmount() # let's use gitcoin's calculated rate for now since they include it in the response usd_value = Price( ZERO) if rawtx['usd_value'] is None else deserialize_price( rawtx['usd_value']) # noqa: E501 rate = Price(ZERO) if usd_value == ZERO else Price(usd_value / amount) raw_txid = rawtx['tx_hash'] tx_type, tx_id = process_gitcoin_txid(key='tx_hash', entry=rawtx) # until we figure out if we can use it https://github.com/gitcoinco/web/issues/9255#issuecomment-874537144 # noqa: E501 clr_round = _calculate_clr_round(timestamp, rawtx) notes = f'Gitcoin grant {grant_id} event' if not clr_round else f'Gitcoin grant {grant_id} event in clr_round {clr_round}' # noqa: E501 return LedgerAction( identifier=1, # whatever -- does not end up in the DB timestamp=timestamp, action_type=LedgerActionType.DONATION_RECEIVED, location=Location.GITCOIN, amount=AssetAmount(amount), asset=asset, rate=rate, rate_asset=A_USD, link=raw_txid, notes=notes, extra_data=GitcoinEventData( tx_id=tx_id, grant_id=grant_id, clr_round=clr_round, tx_type=tx_type, ), )
def deserialize_from_db(cls, entry: TradeDBTuple) -> 'Trade': """May raise: - DeserializationError - UnknownAsset """ return Trade( timestamp=deserialize_timestamp(entry[1]), location=Location.deserialize_from_db(entry[2]), base_asset=Asset(entry[3]), quote_asset=Asset(entry[4]), trade_type=TradeType.deserialize_from_db(entry[5]), amount=deserialize_asset_amount(entry[6]), rate=deserialize_price(entry[7]), fee=deserialize_optional(entry[8], deserialize_fee), fee_currency=deserialize_optional(entry[9], Asset), link=entry[10], notes=entry[11], )
def _deserialize_trade( self, raw_trade: Dict[str, Any], ) -> Trade: """Process a trade user transaction from Bitstamp and deserialize it. Can raise DeserializationError. """ timestamp = deserialize_timestamp_from_bitstamp_date(raw_trade['datetime']) trade_pair_data = self._get_trade_pair_data_from_transaction(raw_trade) base_asset_amount = deserialize_asset_amount( raw_trade[trade_pair_data.base_asset_symbol], ) quote_asset_amount = deserialize_asset_amount( raw_trade[trade_pair_data.quote_asset_symbol], ) rate = deserialize_price(raw_trade[trade_pair_data.pair]) fee_currency = trade_pair_data.quote_asset if base_asset_amount >= ZERO: trade_type = TradeType.BUY else: if quote_asset_amount < 0: raise DeserializationError( f'Unexpected bitstamp trade format. Both base and quote ' f'amounts are negative: {raw_trade}', ) trade_type = TradeType.SELL trade = Trade( timestamp=timestamp, location=Location.BITSTAMP, base_asset=trade_pair_data.base_asset, quote_asset=trade_pair_data.quote_asset, trade_type=trade_type, amount=AssetAmount(abs(base_asset_amount)), rate=rate, fee=deserialize_fee(raw_trade['fee']), fee_currency=fee_currency, link=str(raw_trade['id']), notes='', ) return trade
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - token0_identifier 7 - token1_identifier 8 - amount0 9 - amount1 10 - usd_price 11 - lp_amount """ event_type = EventType.deserialize_from_db(event_tuple[4]) token0 = deserialize_ethereum_token_from_db(identifier=event_tuple[6]) token1 = deserialize_ethereum_token_from_db(identifier=event_tuple[7]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[8]), amount1=deserialize_asset_amount(event_tuple[9]), usd_price=deserialize_price(event_tuple[10]), lp_amount=deserialize_asset_amount(event_tuple[11]), )
def deserialize_from_db(cls: Type[T], timestamp: Timestamp, stringified_json: str) -> T: """May raise: - DeserializationError if something is wrong with reading this from the DB """ try: data = json.loads(stringified_json) except json.decoder.JSONDecodeError as e: raise DeserializationError( f'Could not decode processed accounting event json from the DB due to {str(e)}', ) from e try: pnl_taxable = deserialize_fval(data['pnl_taxable'], name='pnl_taxable', location='processed event decoding') # noqa: E501 pnl_free = deserialize_fval(data['pnl_free'], name='pnl_free', location='processed event decoding') # noqa: E501 if data['cost_basis'] is None: cost_basis = None else: cost_basis = CostBasisInfo.deserialize(data['cost_basis']) event = cls( type=AccountingEventType.deserialize(data['type']), notes=data['notes'], location=Location.deserialize(data['location']), timestamp=timestamp, asset=Asset(data['asset']), free_amount=deserialize_fval(data['free_amount'], name='free_amount', location='processed event decoding'), # noqa: E501 taxable_amount=deserialize_fval(data['taxable_amount'], name='taxable_amount', location='processed event decoding'), # noqa: E501 price=deserialize_price(data['price']), pnl=PNL(free=pnl_free, taxable=pnl_taxable), cost_basis=cost_basis, index=data['index'], extra_data=data['extra_data'], ) event.count_cost_basis_pnl = data['count_cost_basis_pnl'] event.count_entire_amount_spend = data['count_entire_amount_spend'] return event except KeyError as e: raise DeserializationError(f'Could not decode processed accounting event json from the DB due to missing key {str(e)}') from e # noqa: E501
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]: """Queries coinbase pro for trades""" log.debug('Query coinbasepro trade history', start_ts=start_ts, end_ts=end_ts) self.first_connection() trades = [] # first get all orders, to see which product ids we need to query fills for orders = [] for batch in self._paginated_query( endpoint='orders', query_options={'status': 'done'}, ): orders.extend(batch) queried_product_ids = set() for order_entry in orders: product_id = order_entry.get('product_id', None) if product_id is None: msg = ( 'Skipping coinbasepro trade since it lacks a product_id. ' 'Check logs for details') self.msg_aggregator.add_error(msg) log.error( 'Error processing a coinbasepro order.', raw_trade=order_entry, error=msg, ) continue if product_id in queried_product_ids or product_id not in self.available_products: continue # already queried this product id or delisted product id # Now let's get all fills for this product id queried_product_ids.add(product_id) fills = [] for batch in self._paginated_query( endpoint='fills', query_options={'product_id': product_id}, ): fills.extend(batch) try: base_asset, quote_asset = coinbasepro_to_worldpair(product_id) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring the trade.', ) continue for fill_entry in fills: try: timestamp = coinbasepro_deserialize_timestamp( fill_entry, 'created_at') if timestamp < start_ts or timestamp > end_ts: continue # Fee currency seems to always be quote asset # https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501 trades.append( Trade( timestamp=timestamp, location=Location.COINBASEPRO, base_asset=base_asset, quote_asset=quote_asset, trade_type=TradeType.deserialize( fill_entry['side']), amount=deserialize_asset_amount( fill_entry['size']), rate=deserialize_price(fill_entry['price']), fee=deserialize_fee(fill_entry['fee']), fee_currency=quote_asset, link=str(fill_entry['trade_id']) + '_' + fill_entry['order_id'], notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Coinbasepro asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a coinbasepro trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a coinbasepro fill.', raw_trade=fill_entry, error=msg, ) continue return trades, (start_ts, end_ts)
def deserialize_from_db( cls, event_tuple: LiquidityPoolEventDBTuple, ) -> 'LiquidityPoolEvent': """Turns a tuple read from DB into an appropriate LiquidityPoolEvent. May raise a DeserializationError if something is wrong with the DB data Event_tuple index - Schema columns ---------------------------------- 0 - tx_hash 1 - log_index 2 - address 3 - timestamp 4 - type 5 - pool_address 6 - is_token0_unknown 7 - token0_address 8 - token0_symbol 9 - token0_name 10 - token0_decimals 11 - is_token1_unknown 12 - token1_address 13 - token1_symbol 14 - token1_name 15 - token1_decimals 16 - amount0 17 - amount1 18 - usd_price 19 - lp_amount """ db_event_type = event_tuple[4] if db_event_type not in {str(event_type) for event_type in EventType}: raise DeserializationError( f'Failed to deserialize event type. Unknown event: {db_event_type}.', ) if db_event_type == str(EventType.MINT): event_type = EventType.MINT elif db_event_type == str(EventType.BURN): event_type = EventType.BURN else: raise ValueError(f'Unexpected event type case: {db_event_type}.') is_token0_unknown = event_tuple[6] is_token1_unknown = event_tuple[11] token0: Union[EthereumToken, UnknownEthereumToken] token1: Union[EthereumToken, UnknownEthereumToken] if is_token0_unknown: token0 = deserialize_unknown_ethereum_token_from_db( ethereum_address=event_tuple[7], symbol=event_tuple[8], name=event_tuple[9], decimals=event_tuple[10], ) else: token0 = deserialize_ethereum_token_from_db( identifier=event_tuple[8]) if is_token1_unknown: token1 = deserialize_unknown_ethereum_token_from_db( ethereum_address=event_tuple[12], symbol=event_tuple[13], name=event_tuple[14], decimals=event_tuple[15], ) else: token1 = deserialize_ethereum_token_from_db( identifier=event_tuple[13]) return cls( tx_hash=event_tuple[0], log_index=event_tuple[1], address=string_to_ethereum_address(event_tuple[2]), timestamp=deserialize_timestamp(event_tuple[3]), event_type=event_type, pool_address=string_to_ethereum_address(event_tuple[5]), token0=token0, token1=token1, amount0=deserialize_asset_amount(event_tuple[16]), amount1=deserialize_asset_amount(event_tuple[17]), usd_price=deserialize_price(event_tuple[18]), lp_amount=deserialize_asset_amount(event_tuple[19]), )
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() - DeserializationError due to the data being in unexpected format - UnprocessableTradePair due to the pair data being in an unexpected format """ try: trade_type = deserialize_trade_type(poloniex_trade['type']) amount = deserialize_asset_amount(poloniex_trade['amount']) rate = deserialize_price(poloniex_trade['rate']) perc_fee = deserialize_fee(poloniex_trade['fee']) base_currency = asset_from_poloniex( get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex( get_pair_position_str(pair, 'second')) timestamp = deserialize_timestamp_from_poloniex_date( poloniex_trade['date']) except KeyError as e: raise DeserializationError( f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict', ) from e cost = rate * amount if trade_type == TradeType.BUY: fee = Fee(amount * perc_fee) fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = Fee(cost * perc_fee) fee_currency = base_currency else: raise DeserializationError( f'Got unexpected trade type "{trade_type}" for poloniex trade') if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', timestamp=timestamp, order_type=trade_type, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) return Trade( timestamp=timestamp, location=Location.POLONIEX, # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. base_asset=quote_currency, quote_asset=base_currency, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(poloniex_trade['globalTradeID']), )
def _scrape_xratescom_exchange_rates(url: str) -> Dict[Asset, Price]: """ Scrapes x-rates.com website for the exchange rates tables May raise: - RemoteError if we can't query x-rates.com """ log.debug(f'Querying x-rates.com stats: {url}') prices = {} try: response = requests.get(url=url, timeout=DEFAULT_TIMEOUT_TUPLE) except requests.exceptions.RequestException as e: raise RemoteError(f'x-rates.com request {url} failed due to {str(e)}') from e if response.status_code != 200: raise RemoteError( f'x-rates.com request {url} failed with code: {response.status_code}' f' and response: {response.text}', ) soup = BeautifulSoup( response.text, 'html.parser', parse_only=SoupStrainer('table', {'class': 'tablesorter ratesTable'}), ) if soup is None: raise RemoteError('Could not find <table> while parsing x-rates stats page') try: tr = soup.table.tbody.tr except AttributeError as e: raise RemoteError('Could not find first <tr> while parsing x-rates.com page') from e while tr is not None: secondtd = tr.select('td:nth-of-type(2)')[0] try: href = secondtd.a['href'] except (AttributeError, KeyError) as e: raise RemoteError('Could not find a href of 2nd td while parsing x-rates.com page') from e # noqa: E501 parts = href.split('to=') if len(parts) != 2: raise RemoteError(f'Could not find to= in {href} while parsing x-rates.com page') try: to_asset = Asset(parts[1]) if not to_asset.is_fiat(): raise ValueError except (UnknownAsset, ValueError): log.debug(f'Skipping {parts[1]} asset because its not a known fiat asset while parsing x-rates.com page') # noqa: E501 tr = tr.find_next_sibling() continue try: price = deserialize_price(secondtd.a.text) except DeserializationError as e: log.debug(f'Could not parse x-rates.com rate of {to_asset.identifier} due to {str(e)}. Skipping ...') # noqa: E501 tr = tr.find_next_sibling() continue prices[to_asset] = price tr = tr.find_next_sibling() return prices
def trade_from_binance( binance_trade: Dict, binance_symbols_to_pair: Dict[str, BinancePair], location: Location, ) -> Trade: """Turn a binance trade returned from trade history to our common trade history format From the official binance api docs (01/09/18): https://github.com/binance-exchange/binance-official-api-docs/blob/62ff32d27bb32d9cc74d63d547c286bb3c9707ef/rest-api.md#terminology base asset refers to the asset that is the quantity of a symbol. quote asset refers to the asset that is the price of a symbol. Throws: - UnsupportedAsset due to asset_from_binance - DeserializationError due to unexpected format of dict entries - KeyError due to dict entries missing an expected entry """ amount = deserialize_asset_amount(binance_trade['qty']) rate = deserialize_price(binance_trade['price']) if binance_trade['symbol'] not in binance_symbols_to_pair: raise DeserializationError( f'Error reading a {str(location)} trade. Could not find ' f'{binance_trade["symbol"]} in binance_symbols_to_pair', ) binance_pair = binance_symbols_to_pair[binance_trade['symbol']] timestamp = deserialize_timestamp_from_binance(binance_trade['time']) base_asset = asset_from_binance(binance_pair.binance_base_asset) quote_asset = asset_from_binance(binance_pair.binance_quote_asset) if binance_trade['isBuyer']: order_type = TradeType.BUY # e.g. in RDNETH we buy RDN by paying ETH else: order_type = TradeType.SELL fee_currency = asset_from_binance(binance_trade['commissionAsset']) fee = deserialize_fee(binance_trade['commission']) log.debug( f'Processing {str(location)} Trade', amount=amount, rate=rate, timestamp=timestamp, pair=binance_trade['symbol'], base_asset=base_asset, quote=quote_asset, order_type=order_type, commision_asset=binance_trade['commissionAsset'], fee=fee, ) return Trade( timestamp=timestamp, location=location, base_asset=base_asset, quote_asset=quote_asset, trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(binance_trade['id']), )
def _deserialize_trade( self, entry: Dict[str, Any], from_ts: Timestamp, to_ts: Timestamp, ) -> Optional[Trade]: """Deserializes a bitpanda trades result entry to a Trade Returns None and logs error is there is a problem or simpy None if it's not a type of trade we are interested in """ try: if entry['type'] != 'trade' or entry['attributes'][ 'status'] != 'finished': return None time = Timestamp( deserialize_int_from_str( symbol=entry['attributes']['time']['unix'], location='bitpanda trade', )) if time < from_ts or time > to_ts: # should we also stop querying from calling method? # Probably yes but docs don't mention anything about results # being ordered by time so let's be conservative return None cryptocoin_id = entry['attributes']['cryptocoin_id'] crypto_asset = self.cryptocoin_map.get(cryptocoin_id) if crypto_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda cryptocoin ' f'with id {cryptocoin_id} in the mapping. Skipping trade.', ) return None fiat_id = entry['attributes']['fiat_id'] fiat_asset = self.fiat_map.get(fiat_id) if fiat_asset is None: self.msg_aggregator.add_error( f'While deserializing a trade, could not find bitpanda fiat ' f'with id {fiat_id} in the mapping. Skipping trade.', ) return None trade_type = TradeType.deserialize(entry['attributes']['type']) if trade_type in (TradeType.BUY, TradeType.SELL): # you buy crypto with fiat and sell it for fiat base_asset = crypto_asset quote_asset = fiat_asset amount = deserialize_asset_amount( entry['attributes']['amount_cryptocoin']) price = deserialize_price(entry['attributes']['price']) else: self.msg_aggregator.add_error( 'Found bitpanda trade with unknown trade type {trade_type}' ) # noqa: E501 return None trade_id = entry['id'] fee = Fee(ZERO) fee_asset = A_BEST if entry['attributes']['bfc_used'] is True: fee = deserialize_fee( entry['attributes']['best_fee_collection']['attributes'] ['wallet_transaction']['attributes']['fee'], # noqa: E501 ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key {msg} for trade entry' self.msg_aggregator.add_error( f'Error processing bitpanda trade due to {msg}') log.error( 'Error processing bitpanda trade entry', error=msg, entry=entry, ) return None return Trade( timestamp=time, location=Location.BITPANDA, base_asset=base_asset, quote_asset=quote_asset, trade_type=trade_type, amount=amount, rate=price, fee=fee, fee_currency=fee_asset, link=trade_id, )
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]: """Queries gemini for trades """ log.debug('Query gemini trade history', start_ts=start_ts, end_ts=end_ts) trades = [] gemini_trades = [] for symbol in self.symbols: gemini_trades = self._get_trades_for_symbol( symbol=symbol, start_ts=start_ts, end_ts=end_ts, ) for entry in gemini_trades: try: timestamp = deserialize_timestamp(entry['timestamp']) if timestamp > end_ts: break base, quote = gemini_symbol_to_base_quote(symbol) trades.append(Trade( timestamp=timestamp, location=Location.GEMINI, base_asset=base, quote_asset=quote, trade_type=TradeType.deserialize(entry['type']), amount=deserialize_asset_amount(entry['amount']), rate=deserialize_price(entry['price']), fee=deserialize_fee(entry['fee_amount']), fee_currency=asset_from_gemini(entry['fee_currency']), link=str(entry['tid']), notes='', )) except UnprocessableTradePair as e: self.msg_aggregator.add_warning( f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown Gemini asset {e.asset_name}. ' f'Ignoring the trade.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Failed to deserialize a gemini trade. ' 'Check logs for details. Ignoring it.', ) log.error( 'Error processing a gemini trade.', raw_trade=entry, error=msg, ) continue return trades, (start_ts, end_ts)
- UnsupportedAsset """ try: timestamp = deserialize_timestamp(raw_result['createdAt']) trade_type = TradeType.BUY if raw_result[ 'side'] == 'buy' else TradeType.SELL # amount_key = 'size' if case == KucoinCase.TRADES else 'amount' # amount = deserialize_asset_amount(raw_result['size']) fee = deserialize_fee(raw_result['fee']) trade_pair_symbol = raw_result['symbol'] base_asset, quote_asset = deserialize_trade_pair(trade_pair_symbol) if case == KucoinCase.TRADES: fee_currency_symbol = raw_result['feeCurrency'] fee_currency = asset_from_kucoin(fee_currency_symbol) amount = deserialize_asset_amount(raw_result['size']) rate = deserialize_price(raw_result['price']) # new trades have the timestamp in ms timestamp = Timestamp(int(timestamp / 1000)) trade_id = raw_result['tradeId'] else: # old v1 trades amount = deserialize_asset_amount(raw_result['amount']) fee_currency = quote_asset if trade_type == TradeType.SELL else base_asset rate = deserialize_price(raw_result['dealPrice']) trade_id = raw_result['id'] except KeyError as e: raise DeserializationError(f'Missing key: {str(e)}.') from e trade = Trade( timestamp=timestamp, location=Location.KUCOIN,
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade: """Turn a kraken trade returned from kraken trade history to our common trade history format - Can raise UnknownAsset due to kraken_to_world_pair - Can raise UnprocessableTradePair due to kraken_to_world_pair - Can raise DeserializationError due to dict entries not being as expected - Can raise KeyError due to dict entries missing an expected entry """ base_asset, quote_asset = kraken_to_world_pair(kraken_trade['pair']) timestamp = deserialize_timestamp_from_kraken(kraken_trade['time']) amount = deserialize_asset_amount(kraken_trade['vol']) cost = deserialize_price(kraken_trade['cost']) fee = deserialize_fee(kraken_trade['fee']) order_type = deserialize_trade_type(kraken_trade['type']) rate = deserialize_price(kraken_trade['price']) # pylint does not seem to see that Price is essentially FVal if not cost.is_close(amount * rate): # pylint: disable=no-member log.warning(f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade') log.debug( 'Processing kraken Trade', timestamp=timestamp, order_type=order_type, kraken_pair=kraken_trade['pair'], base_asset=base_asset, quote_asset=quote_asset, amount=amount, cost=cost, fee=fee, rate=rate, ) # Kraken trades can have the same ordertxid and postxid for different trades .. # Also note postxid is optional and can be missing # The only thing that could differentiate them is timestamps in the milliseconds range # For example here are parts of two different kraken_trade: # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT', # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955) # {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT', # 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005) # # In order to counter this for the unique exchange trade link we are going # to use a concatenation of the above exchange_uuid = ( str(kraken_trade['ordertxid']) + str(kraken_trade.get('postxid', '')) + # postxid is optional str(kraken_trade['time']) ) return Trade( timestamp=timestamp, location=Location.KRAKEN, base_asset=base_asset, quote_asset=quote_asset, trade_type=order_type, amount=amount, rate=rate, fee=fee, fee_currency=quote_asset, link=exchange_uuid, )
def query_and_store_historical_data( self, from_asset: Asset, to_asset: Asset, timestamp: Timestamp, ) -> None: """ Get historical hour price data from cryptocompare and populate the global DB - May raise RemoteError if there is a problem reaching the cryptocompare server or with reading the response returned by the server - May raise UnsupportedAsset if from/to asset is not supported by cryptocompare """ log.debug( 'Retrieving historical hour price data from cryptocompare', from_asset=from_asset, to_asset=to_asset, timestamp=timestamp, ) now_ts = ts_now() # save time at start of the query, in case the query does not complete due to rate limit self.last_histohour_query_ts = now_ts range_result = GlobalDBHandler().get_historical_price_range( from_asset=from_asset, to_asset=to_asset, source=HistoricalPriceOracle.CRYPTOCOMPARE, ) if range_result is not None: first_cached_ts, last_cached_ts = range_result if timestamp > last_cached_ts: # We have a cache but the requested timestamp does not hit it new_data = self._get_histohour_data_for_range( from_asset=from_asset, to_asset=to_asset, from_timestamp=now_ts, to_timestamp=last_cached_ts, ) else: # only other possibility, timestamp < cached start_time new_data = self._get_histohour_data_for_range( from_asset=from_asset, to_asset=to_asset, from_timestamp=first_cached_ts, to_timestamp=Timestamp(0), ) else: new_data = self._get_histohour_data_for_range( from_asset=from_asset, to_asset=to_asset, from_timestamp=now_ts, to_timestamp=Timestamp(0), ) calculated_history = list(new_data) if len(calculated_history) == 0: return # Let's always check for data sanity for the hourly prices. _check_hourly_data_sanity(calculated_history, from_asset, to_asset) # Turn them into the format we will enter in the DB prices = [] for entry in calculated_history: try: price = Price((deserialize_price(entry['high']) + deserialize_price(entry['low'])) / 2) # noqa: E501 if price == Price(ZERO): continue # don't write zero prices prices.append(HistoricalPrice( from_asset=from_asset, to_asset=to_asset, source=HistoricalPriceOracle.CRYPTOCOMPARE, timestamp=Timestamp(entry['time']), price=price, )) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' log.error( f'{msg}. Error getting price entry from cryptocompare histohour ' f'price results. Skipping entry.', ) continue GlobalDBHandler().add_historical_prices(prices) self.last_histohour_query_ts = ts_now() # also save when last query finished