def process_polo_loans(
msg_aggregator: MessagesAggregator,
data: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Loan]:
"""Takes in the list of loans from poloniex as returned by the return_lending_history
api call, processes it and returns it into our loan format
"""
new_data = []
for loan in reversed(data):
log.debug('processing poloniex loan', **make_sensitive(loan))
try:
close_time = deserialize_timestamp_from_poloniex_date(
loan['close'])
open_time = deserialize_timestamp_from_poloniex_date(loan['open'])
if open_time < start_ts:
continue
if close_time > end_ts:
continue
our_loan = Loan(
location=Location.POLONIEX,
open_time=open_time,
close_time=close_time,
currency=asset_from_poloniex(loan['currency']),
fee=deserialize_fee(loan['fee']),
earned=deserialize_asset_amount(loan['earned']),
amount_lent=deserialize_asset_amount(loan['amount']),
)
except UnsupportedAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unsupported asset'
f' {e.asset_name}. Ignoring it.', )
continue
except UnknownAsset as e:
msg_aggregator.add_warning(
f'Found poloniex loan with unknown asset'
f' {e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
msg_aggregator.add_error(
'Deserialization error while reading a poloniex loan. Check '
'logs for more details. Ignoring it.', )
log.error(
'Deserialization error while reading a poloniex loan',
loan=loan,
error=msg,
)
continue
new_data.append(our_loan)
new_data.sort(key=lambda loan: loan.open_time)
return new_data
def return_trade_history(
self,
start: Timestamp,
end: Timestamp,
) -> Dict[str, List[Dict[str, Any]]]:
"""If `currency_pair` is all, then it returns a dictionary with each key
being a pair and each value a list of trades. If `currency_pair` is a specific
pair then a list is returned"""
limit = 10000
pair = 'all'
data: DefaultDict[str, List[Dict[str, Any]]] = defaultdict(list)
while True:
new_data = self.api_query_dict('returnTradeHistory', {
'currencyPair': pair,
'start': start,
'end': end,
'limit': limit,
})
results_length = 0
for _, v in new_data.items():
results_length += len(v)
if data == {} and results_length < limit:
return new_data # simple case - only one query needed
latest_ts = start
# add results to data and prepare for next query
for market, trades in new_data.items():
existing_ids = {x['globalTradeID'] for x in data['market']}
for trade in trades:
try:
timestamp = deserialize_timestamp_from_poloniex_date(
trade['date'])
latest_ts = max(latest_ts, timestamp)
# since we query again from last ts seen make sure no duplicates make it in
if trade['globalTradeID'] not in existing_ids:
data[market].append(trade)
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_warning(
'Error deserializing a poloniex trade. Check the logs for details',
)
log.error(
'Error deserializing poloniex trade',
trade=trade,
error=msg,
)
continue
if results_length < limit:
break # last query has less than limit. We are done.
# otherwise we query again from the last ts seen in the last result
start = latest_ts
continue
return data
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> Tuple[List[Trade], Tuple[Timestamp, Timestamp]]:
raw_data = self.return_trade_history(
start=start_ts,
end=end_ts,
)
results_length = 0
for _, v in raw_data.items():
results_length += len(v)
log.debug('Poloniex trade history query', results_num=results_length)
our_trades = []
for pair, trades in raw_data.items():
for trade in trades:
category = trade.get('category', None)
try:
if category in ('exchange', 'settlement'):
timestamp = deserialize_timestamp_from_poloniex_date(trade['date'])
if timestamp < start_ts or timestamp > end_ts:
continue
our_trades.append(trade_from_poloniex(trade, TradePair(pair)))
elif category == 'marginTrade':
# We don't take poloniex margin trades into account at the moment
continue
else:
self.msg_aggregator.add_error(
f'Error deserializing a poloniex trade. Unknown trade '
f'category {category} found.',
)
continue
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found poloniex trade with unsupported asset'
f' {e.asset_name}. Ignoring it.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found poloniex trade with unknown asset'
f' {e.asset_name}. Ignoring it.',
)
continue
except (UnprocessableTradePair, DeserializationError) as e:
self.msg_aggregator.add_error(
'Error deserializing a poloniex trade. Check the logs '
'and open a bug report.',
)
log.error(
'Error deserializing poloniex trade',
trade=trade,
error=str(e),
)
continue
return our_trades, (start_ts, end_ts)
def query_loan_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
from_csv: Optional[bool] = False,
) -> List:
"""
WARNING: Querying from returnLendingHistory endpoint instead of reading from
the CSV file can potentially return unexpected/wrong results.
That is because the `returnLendingHistory` endpoint has a hidden limit
of 12660 results. In our code we use the limit of 12000 but poloniex may change
the endpoint to have a lower limit at which case this code will break.
To be safe compare results of both CSV and endpoint to make sure they agree!
"""
try:
if from_csv:
return self.parse_loan_csv()
except (OSError, csv.Error):
pass
loans_query_return_limit = 12000
result = self.return_lending_history(
start_ts=start_ts,
end_ts=end_ts,
limit=loans_query_return_limit,
)
data = list(result)
log.debug('Poloniex loan history query', results_num=len(data))
# since I don't think we have any guarantees about order of results
# using a set of loan ids is one way to make sure we get no duplicates
# if poloniex can guarantee me that the order is going to be ascending/descending
# per open/close time then this can be improved
id_set = set()
while len(result) == loans_query_return_limit:
# Find earliest timestamp to re-query the next batch
min_ts = end_ts
for loan in result:
ts = deserialize_timestamp_from_poloniex_date(loan['close'])
min_ts = min(min_ts, ts)
id_set.add(loan['id'])
result = self.return_lending_history(
start_ts=start_ts,
end_ts=min_ts,
limit=loans_query_return_limit,
)
log.debug('Poloniex loan history query', results_num=len(result))
for loan in result:
if loan['id'] not in id_set:
data.append(loan)
return data
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)