def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
"""
trade_type = trade_type_from_string(poloniex_trade['type'])
amount = FVal(poloniex_trade['amount'])
rate = FVal(poloniex_trade['rate'])
perc_fee = FVal(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = createTimeStamp(poloniex_trade['date'], formatstr="%Y-%m-%d %H:%M:%S")
cost = rate * amount
if trade_type == TradeType.BUY:
fee = amount * perc_fee
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = cost * perc_fee
fee_currency = base_currency
else:
raise ValueError('Got unexpected trade type "{}" for poloniex trade'.format(trade_type))
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location='poloniex',
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
def parse_loan_csv(self) -> List:
"""Parses (if existing) the lendingHistory.csv and returns the history in a list
It can throw OSError, IOError if the file does not exist and csv.Error if
the file is not proper CSV"""
# the default filename, and should be (if at all) inside the data directory
path = os.path.join(self.db.user_data_dir, "lendingHistory.csv")
lending_history = []
with open(path, 'r') as csvfile:
history = csv.reader(csvfile, delimiter=',', quotechar='|')
next(history) # skip header row
for row in history:
try:
lending_history.append({
'currency': asset_from_poloniex(row[0]),
'earned': FVal(row[6]),
'amount': FVal(row[2]),
'fee': FVal(row[5]),
'open': row[7],
'close': row[8],
})
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found loan with asset {e.asset_name}. Ignoring it.',
)
continue
return lending_history
def query_balances(
self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
resp = self.api_query_dict('returnCompleteBalances',
{"account": "all"})
except RemoteError as e:
msg = ('Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e))
log.error(msg)
return None, msg
balances = {}
for poloniex_asset, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if (available != FVal(0) or on_orders != FVal(0)):
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
log.error(
f'Unexpected poloniex asset type. Expected string '
f' but got {type(poloniex_asset)}', )
self.msg_aggregator.add_error(
'Found poloniex asset entry with non-string type. '
' Ignoring its balance query.', )
continue
entry = {}
entry['amount'] = available + on_orders
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing poloniex balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
usd_value = entry['amount'] * usd_price
entry['usd_value'] = usd_value
balances[asset] = entry
log.debug(
'Poloniex balance query',
sensitive_log=True,
currency=asset,
amount=entry['amount'],
usd_value=usd_value,
)
return balances, ''
def test_poloniex_assets_are_known(poloniex):
currencies = poloniex.return_currencies()
for poloniex_asset in currencies.keys():
try:
_ = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset:
assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS
def process_polo_loans(
msg_aggregator: MessagesAggregator,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Loan]:
"""Takes in the list of loans from poloniex as returned by the return_lending_history
api call, processes it and returns it into our loan format
"""
new_data = []
for loan in reversed(data):
log.debug('processing poloniex loan', **make_sensitive(loan))
try:
close_time = deserialize_timestamp_from_poloniex_date(
loan['close'])
open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
if open_time < start_ts:
continue
if close_time > end_ts:
continue
our_loan = Loan(
location=Location.POLONIEX,
open_time=open_time,
close_time=close_time,
currency=asset_from_poloniex(loan['currency']),
fee=deserialize_fee(loan['fee']),
earned=deserialize_asset_amount(loan['earned']),
amount_lent=deserialize_asset_amount(loan['amount']),
)
except UnsupportedAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
msg_aggregator.add_error(
'Deserialization error while reading a poloniex loan. Check '
'logs for more details. Ignoring it.', )
log.error(
'Deserialization error while reading a poloniex loan',
loan=loan,
error=msg,
)
continue
new_data.append(our_loan)
new_data.sort(key=lambda loan: loan.open_time)
return new_data
def test_poloniex_assets_are_known(poloniex):
currencies = poloniex.return_currencies()
for poloniex_asset in currencies.keys():
try:
_ = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset:
assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS
except UnknownAsset as e:
test_warnings.warn(UserWarning(
f'Found unknown asset {e.asset_name} in Poloniex. Support for it has to be added',
))
def _deserialize_asset_movement(
self,
movement_type: AssetMovementCategory,
movement_data: Dict[str, Any],
) -> Optional[AssetMovement]:
"""Processes a single deposit/withdrawal from polo and deserializes it
Can log error/warning and return None if something went wrong at deserialization
"""
try:
if movement_type == AssetMovementCategory.DEPOSIT:
fee = Fee(ZERO)
uid_key = 'depositNumber'
else:
fee = deserialize_fee(movement_data['fee'])
uid_key = 'withdrawalNumber'
asset = asset_from_poloniex(movement_data['currency'])
return AssetMovement(
location=Location.POLONIEX,
category=movement_type,
timestamp=deserialize_timestamp(movement_data['timestamp']),
asset=asset,
amount=deserialize_asset_amount(movement_data['amount']),
fee_asset=asset,
fee=fee,
link=str(movement_data[uid_key]),
)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unsupported poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found {str(movement_type)} of unknown poloniex asset '
f'{e.asset_name}. Ignoring it.',
)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a poloniex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'{str(movement_type)}: {movement_data}. Error was: {str(e)}',
)
return None
def test_poloniex_assets_are_known(poloniex):
unsupported_assets = set(UNSUPPORTED_POLONIEX_ASSETS)
common_items = unsupported_assets.intersection(
set(WORLD_TO_POLONIEX.values()))
assert not common_items, f'Poloniex assets {common_items} should not be unsupported'
currencies = poloniex.return_currencies()
for poloniex_asset in currencies.keys():
try:
_ = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset:
assert poloniex_asset in UNSUPPORTED_POLONIEX_ASSETS
except UnknownAsset as e:
test_warnings.warn(
UserWarning(
f'Found unknown asset {e.asset_name} in Poloniex. Support for it has to be added',
))
def query_balances(self) -> Tuple[Optional[Dict[Asset, Dict[str, Any]]], str]:
try:
resp = self.api_query_dict('returnCompleteBalances', {"account": "all"})
except (RemoteError, PoloniexError) as e:
msg = (
'Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e)
)
log.error(msg)
return None, msg
balances = dict()
for poloniex_asset, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if (available != FVal(0) or on_orders != FVal(0)):
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.',
)
continue
entry = {}
entry['amount'] = available + on_orders
usd_price = Inquirer().find_usd_price(asset=asset)
usd_value = entry['amount'] * usd_price
entry['usd_value'] = usd_value
balances[asset] = entry
log.debug(
'Poloniex balance query',
sensitive_log=True,
currency=asset,
amount=entry['amount'],
usd_value=usd_value,
)
return balances, ''
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def query_balances(self) -> ExchangeQueryBalances:
try:
resp = self.api_query_dict('returnCompleteBalances',
{"account": "all"})
except RemoteError as e:
msg = ('Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e))
log.error(msg)
return None, msg
assets_balance: Dict[Asset, Balance] = {}
for poloniex_asset, v in resp.items():
try:
available = deserialize_asset_amount(v['available'])
on_orders = deserialize_asset_amount(v['onOrders'])
except DeserializationError as e:
self.msg_aggregator.add_error(
f'Could not deserialize amount from poloniex due to '
f'{str(e)}. Ignoring its balance query.', )
continue
if available != ZERO or on_orders != ZERO:
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
log.error(
f'Unexpected poloniex asset type. Expected string '
f' but got {type(poloniex_asset)}', )
self.msg_aggregator.add_error(
'Found poloniex asset entry with non-string type. '
' Ignoring its balance query.', )
continue
if asset == A_LEND: # poloniex mistakenly returns LEND balances
continue # https://github.com/rotki/rotki/issues/2530
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing poloniex balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
amount = available + on_orders
usd_value = amount * usd_price
assets_balance[asset] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Poloniex balance query',
currency=asset,
amount=amount,
usd_value=usd_value,
)
return assets_balance, ''
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_dict(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
if cache is None:
result = self.returnDepositsWithdrawals(start_ts, end_ts)
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
else:
result = cache
log.debug(
'Poloniex deposits/withdrawal query',
results_num=len(result['withdrawals']) + len(result['deposits']),
)
movements = list()
for withdrawal in result['withdrawals']:
try:
movements.append(AssetMovement(
exchange='poloniex',
category='withdrawal',
timestamp=withdrawal['timestamp'],
asset=asset_from_poloniex(withdrawal['currency']),
amount=FVal(withdrawal['amount']),
fee=Fee(FVal(withdrawal['fee'])),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
for deposit in result['deposits']:
try:
movements.append(AssetMovement(
exchange='poloniex',
category='deposit',
timestamp=deposit['timestamp'],
asset=asset_from_poloniex(deposit['currency']),
amount=FVal(deposit['amount']),
fee=Fee(ZERO),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
return movements
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List[AssetMovement]:
with self.lock:
cache = self.check_trades_cache_dict(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
if cache is None:
result = self.returnDepositsWithdrawals(start_ts, end_ts)
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
else:
result = cache
log.debug(
'Poloniex deposits/withdrawal query',
results_num=len(result['withdrawals']) + len(result['deposits']),
)
movements = list()
for withdrawal in result['withdrawals']:
try:
movements.append(
AssetMovement(
exchange='poloniex',
category='withdrawal',
timestamp=deserialize_timestamp(
withdrawal['timestamp']),
asset=asset_from_poloniex(withdrawal['currency']),
amount=deserialize_asset_amount(withdrawal['amount']),
fee=deserialize_fee(withdrawal['fee']),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found withdrawal of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except DeserializationError as e:
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'withdrawal: {withdrawal}. Error was: {str(e)}', )
self.msg_aggregator.add_warning(
f'Unexpected data encountered during deserialization of a poloniex '
f'withdrawal. Check logs for details and open a bug report.',
)
for deposit in result['deposits']:
try:
movements.append(
AssetMovement(
exchange='poloniex',
category='deposit',
timestamp=deserialize_timestamp(deposit['timestamp']),
asset=asset_from_poloniex(deposit['currency']),
amount=deserialize_asset_amount(FVal(
deposit['amount'])),
fee=Fee(ZERO),
))
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unsupported poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found deposit of unknown poloniex asset {e.asset_name}. Ignoring it.',
)
continue
except DeserializationError as e:
log.error(
f'Unexpected data encountered during deserialization of poloniex '
f'deposit: {deposit}. Error was: {str(e)}', )
self.msg_aggregator.add_warning(
f'Unexpected data encountered during deserialization of a poloniex '
f'deposit. Check logs for details and open a bug report.',
)
return movements