def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() """ trade_type = trade_type_from_string(poloniex_trade['type']) amount = FVal(poloniex_trade['amount']) rate = FVal(poloniex_trade['rate']) perc_fee = FVal(poloniex_trade['fee']) base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second')) timestamp = createTimeStamp(poloniex_trade['date'], formatstr="%Y-%m-%d %H:%M:%S") cost = rate * amount if trade_type == TradeType.BUY: fee = amount * perc_fee fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = cost * perc_fee fee_currency = base_currency else: raise ValueError('Got unexpected trade type "{}" for poloniex trade'.format(trade_type)) if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', sensitive_log=True, timestamp=timestamp, order_type=trade_type, pair=pair, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) # Use the converted assets in our pair pair = trade_pair_from_assets(base_currency, quote_currency) # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. pair = invert_pair(pair) return Trade( timestamp=timestamp, location='poloniex', pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, )
def parse_loan_csv(self) -> List: """Parses (if existing) the lendingHistory.csv and returns the history in a list It can throw OSError, IOError if the file does not exist and csv.Error if the file is not proper CSV""" # the default filename, and should be (if at all) inside the data directory path = os.path.join(self.db.user_data_dir, "lendingHistory.csv") lending_history = [] with open(path, 'r') as csvfile: history = csv.reader(csvfile, delimiter=',', quotechar='|') next(history) # skip header row for row in history: try: lending_history.append({ 'currency': asset_from_poloniex(row[0]), 'earned': FVal(row[6]), 'amount': FVal(row[2]), 'fee': FVal(row[5]), 'open': row[7], 'close': row[8], }) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found loan with asset {e.asset_name}. Ignoring it.', ) continue return lending_history
def query_balances( self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]: try: resp = self.api_query_dict('returnCompleteBalances', {"account": "all"}) except RemoteError as e: msg = ('Poloniex API request failed. Could not reach poloniex due ' 'to {}'.format(e)) log.error(msg) return None, msg balances = {} for poloniex_asset, v in resp.items(): available = FVal(v['available']) on_orders = FVal(v['onOrders']) if (available != FVal(0) or on_orders != FVal(0)): try: asset = asset_from_poloniex(poloniex_asset) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found unsupported poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except DeserializationError: log.error( f'Unexpected poloniex asset type. Expected string ' f' but got {type(poloniex_asset)}', ) self.msg_aggregator.add_error( 'Found poloniex asset entry with non-string type. ' ' Ignoring its balance query.', ) continue entry = {} entry['amount'] = available + on_orders try: usd_price = Inquirer().find_usd_price(asset=asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing poloniex balance entry due to inability to ' f'query USD price: {str(e)}. Skipping balance entry', ) continue usd_value = entry['amount'] * usd_price entry['usd_value'] = usd_value balances[asset] = entry log.debug( 'Poloniex balance query', sensitive_log=True, currency=asset, amount=entry['amount'], usd_value=usd_value, ) return balances, ''
def test_poloniex_assets_are_known(poloniex): currencies = poloniex.return_currencies() for poloniex_asset in currencies.keys(): try: _ = asset_from_poloniex(poloniex_asset) except UnsupportedAsset: assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS
def process_polo_loans( msg_aggregator: MessagesAggregator, data: List[Dict], start_ts: Timestamp, end_ts: Timestamp, ) -> List[Loan]: """Takes in the list of loans from poloniex as returned by the return_lending_history api call, processes it and returns it into our loan format """ new_data = [] for loan in reversed(data): log.debug('processing poloniex loan', **make_sensitive(loan)) try: close_time = deserialize_timestamp_from_poloniex_date( loan['close']) open_time = deserialize_timestamp_from_poloniex_date(loan['open']) if open_time < start_ts: continue if close_time > end_ts: continue our_loan = Loan( location=Location.POLONIEX, open_time=open_time, close_time=close_time, currency=asset_from_poloniex(loan['currency']), fee=deserialize_fee(loan['fee']), earned=deserialize_asset_amount(loan['earned']), amount_lent=deserialize_asset_amount(loan['amount']), ) except UnsupportedAsset as e: msg_aggregator.add_warning( f'Found poloniex loan with unsupported asset' f' {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: msg_aggregator.add_warning( f'Found poloniex loan with unknown asset' f' {e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' msg_aggregator.add_error( 'Deserialization error while reading a poloniex loan. Check ' 'logs for more details. Ignoring it.', ) log.error( 'Deserialization error while reading a poloniex loan', loan=loan, error=msg, ) continue new_data.append(our_loan) new_data.sort(key=lambda loan: loan.open_time) return new_data
def test_poloniex_assets_are_known(poloniex): currencies = poloniex.return_currencies() for poloniex_asset in currencies.keys(): try: _ = asset_from_poloniex(poloniex_asset) except UnsupportedAsset: assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS except UnknownAsset as e: test_warnings.warn(UserWarning( f'Found unknown asset {e.asset_name} in Poloniex. Support for it has to be added', ))
def _deserialize_asset_movement( self, movement_type: AssetMovementCategory, movement_data: Dict[str, Any], ) -> Optional[AssetMovement]: """Processes a single deposit/withdrawal from polo and deserializes it Can log error/warning and return None if something went wrong at deserialization """ try: if movement_type == AssetMovementCategory.DEPOSIT: fee = Fee(ZERO) uid_key = 'depositNumber' else: fee = deserialize_fee(movement_data['fee']) uid_key = 'withdrawalNumber' asset = asset_from_poloniex(movement_data['currency']) return AssetMovement( location=Location.POLONIEX, category=movement_type, timestamp=deserialize_timestamp(movement_data['timestamp']), asset=asset, amount=deserialize_asset_amount(movement_data['amount']), fee_asset=asset, fee=fee, link=str(movement_data[uid_key]), ) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unsupported poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found {str(movement_type)} of unknown poloniex asset ' f'{e.asset_name}. Ignoring it.', ) except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a poloniex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of poloniex ' f'{str(movement_type)}: {movement_data}. Error was: {str(e)}', ) return None
def test_poloniex_assets_are_known(poloniex): unsupported_assets = set(UNSUPPORTED_POLONIEX_ASSETS) common_items = unsupported_assets.intersection( set(WORLD_TO_POLONIEX.values())) assert not common_items, f'Poloniex assets {common_items} should not be unsupported' currencies = poloniex.return_currencies() for poloniex_asset in currencies.keys(): try: _ = asset_from_poloniex(poloniex_asset) except UnsupportedAsset: assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS except UnknownAsset as e: test_warnings.warn( UserWarning( f'Found unknown asset {e.asset_name} in Poloniex. Support for it has to be added', ))
def query_balances(self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]: try: resp = self.api_query_dict('returnCompleteBalances', {"account": "all"}) except (RemoteError, PoloniexError) as e: msg = ( 'Poloniex API request failed. Could not reach poloniex due ' 'to {}'.format(e) ) log.error(msg) return None, msg balances = dict() for poloniex_asset, v in resp.items(): available = FVal(v['available']) on_orders = FVal(v['onOrders']) if (available != FVal(0) or on_orders != FVal(0)): try: asset = asset_from_poloniex(poloniex_asset) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found unsupported poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue entry = {} entry['amount'] = available + on_orders usd_price = Inquirer().find_usd_price(asset=asset) usd_value = entry['amount'] * usd_price entry['usd_value'] = usd_value balances[asset] = entry log.debug( 'Poloniex balance query', sensitive_log=True, currency=asset, amount=entry['amount'], usd_value=usd_value, ) return balances, ''
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade: """Turn a poloniex trade returned from poloniex trade history to our common trade history format Throws: - UnsupportedAsset due to asset_from_poloniex() - DeserializationError due to the data being in unexpected format - UnprocessableTradePair due to the pair data being in an unexpected format """ try: trade_type = deserialize_trade_type(poloniex_trade['type']) amount = deserialize_asset_amount(poloniex_trade['amount']) rate = deserialize_price(poloniex_trade['rate']) perc_fee = deserialize_fee(poloniex_trade['fee']) base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first')) quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second')) timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date']) except KeyError as e: raise DeserializationError( f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict', ) cost = rate * amount if trade_type == TradeType.BUY: fee = Fee(amount * perc_fee) fee_currency = quote_currency elif trade_type == TradeType.SELL: fee = Fee(cost * perc_fee) fee_currency = base_currency else: raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade') if poloniex_trade['category'] == 'settlement': if trade_type == TradeType.BUY: trade_type = TradeType.SETTLEMENT_BUY else: trade_type = TradeType.SETTLEMENT_SELL log.debug( 'Processing poloniex Trade', sensitive_log=True, timestamp=timestamp, order_type=trade_type, pair=pair, base_currency=base_currency, quote_currency=quote_currency, amount=amount, fee=fee, rate=rate, ) # Use the converted assets in our pair pair = trade_pair_from_assets(base_currency, quote_currency) # Since in Poloniex the base currency is the cost currency, iow in poloniex # for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it # into the Rotkehlchen way which is following the base/quote approach. pair = invert_pair(pair) return Trade( timestamp=timestamp, location=Location.POLONIEX, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=fee, fee_currency=fee_currency, link=str(poloniex_trade['globalTradeID']), )
def query_balances(self) -> ExchangeQueryBalances: try: resp = self.api_query_dict('returnCompleteBalances', {"account": "all"}) except RemoteError as e: msg = ('Poloniex API request failed. Could not reach poloniex due ' 'to {}'.format(e)) log.error(msg) return None, msg assets_balance: Dict[Asset, Balance] = {} for poloniex_asset, v in resp.items(): try: available = deserialize_asset_amount(v['available']) on_orders = deserialize_asset_amount(v['onOrders']) except DeserializationError as e: self.msg_aggregator.add_error( f'Could not deserialize amount from poloniex due to ' f'{str(e)}. Ignoring its balance query.', ) continue if available != ZERO or on_orders != ZERO: try: asset = asset_from_poloniex(poloniex_asset) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found unsupported poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found unknown poloniex asset {e.asset_name}. ' f' Ignoring its balance query.', ) continue except DeserializationError: log.error( f'Unexpected poloniex asset type. Expected string ' f' but got {type(poloniex_asset)}', ) self.msg_aggregator.add_error( 'Found poloniex asset entry with non-string type. ' ' Ignoring its balance query.', ) continue if asset == A_LEND: # poloniex mistakenly returns LEND balances continue # https://github.com/rotki/rotki/issues/2530 try: usd_price = Inquirer().find_usd_price(asset=asset) except RemoteError as e: self.msg_aggregator.add_error( f'Error processing poloniex balance entry due to inability to ' f'query USD price: {str(e)}. Skipping balance entry', ) continue amount = available + on_orders usd_value = amount * usd_price assets_balance[asset] = Balance( amount=amount, usd_value=usd_value, ) log.debug( 'Poloniex balance query', currency=asset, amount=amount, usd_value=usd_value, ) return assets_balance, ''
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_dict( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) if cache is None: result = self.returnDepositsWithdrawals(start_ts, end_ts) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) else: result = cache log.debug( 'Poloniex deposits/withdrawal query', results_num=len(result['withdrawals']) + len(result['deposits']), ) movements = list() for withdrawal in result['withdrawals']: try: movements.append(AssetMovement( exchange='poloniex', category='withdrawal', timestamp=withdrawal['timestamp'], asset=asset_from_poloniex(withdrawal['currency']), amount=FVal(withdrawal['amount']), fee=Fee(FVal(withdrawal['fee'])), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue for deposit in result['deposits']: try: movements.append(AssetMovement( exchange='poloniex', category='deposit', timestamp=deposit['timestamp'], asset=asset_from_poloniex(deposit['currency']), amount=FVal(deposit['amount']), fee=Fee(ZERO), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue return movements
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, end_at_least_ts: Timestamp, ) -> List[AssetMovement]: with self.lock: cache = self.check_trades_cache_dict( start_ts, end_at_least_ts, special_name='deposits_withdrawals', ) if cache is None: result = self.returnDepositsWithdrawals(start_ts, end_ts) with self.lock: self.update_trades_cache( result, start_ts, end_ts, special_name='deposits_withdrawals', ) else: result = cache log.debug( 'Poloniex deposits/withdrawal query', results_num=len(result['withdrawals']) + len(result['deposits']), ) movements = list() for withdrawal in result['withdrawals']: try: movements.append( AssetMovement( exchange='poloniex', category='withdrawal', timestamp=deserialize_timestamp( withdrawal['timestamp']), asset=asset_from_poloniex(withdrawal['currency']), amount=deserialize_asset_amount(withdrawal['amount']), fee=deserialize_fee(withdrawal['fee']), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue except DeserializationError as e: log.error( f'Unexpected data encountered during deserialization of poloniex ' f'withdrawal: {withdrawal}. Error was: {str(e)}', ) self.msg_aggregator.add_warning( f'Unexpected data encountered during deserialization of a poloniex ' f'withdrawal. Check logs for details and open a bug report.', ) for deposit in result['deposits']: try: movements.append( AssetMovement( exchange='poloniex', category='deposit', timestamp=deserialize_timestamp(deposit['timestamp']), asset=asset_from_poloniex(deposit['currency']), amount=deserialize_asset_amount(FVal( deposit['amount'])), fee=Fee(ZERO), )) except UnsupportedAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.', ) continue except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.', ) continue except DeserializationError as e: log.error( f'Unexpected data encountered during deserialization of poloniex ' f'deposit: {deposit}. Error was: {str(e)}', ) self.msg_aggregator.add_warning( f'Unexpected data encountered during deserialization of a poloniex ' f'deposit. Check logs for details and open a bug report.', ) return movements