def test_deserialize_trade_type():
assert deserialize_trade_type('buy') == TradeType.BUY
assert deserialize_trade_type('sell') == TradeType.SELL
assert deserialize_trade_type('settlement_buy') == TradeType.SETTLEMENT_BUY
assert deserialize_trade_type('settlement_sell') == TradeType.SETTLEMENT_SELL
assert len(list(TradeType)) == 4
with pytest.raises(DeserializationError):
deserialize_trade_type('dsad')
with pytest.raises(DeserializationError):
deserialize_trade_type(None)
with pytest.raises(DeserializationError):
deserialize_trade_type(1)
def trade_from_ftx(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns an FTX transaction into a rotki Trade.
May raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected key
"""
# In the case of perpetuals and futures this fields can be None
if raw_trade.get('baseCurrency', None) is None:
return None
if raw_trade.get('quoteCurrency', None) is None:
return None
timestamp = deserialize_timestamp_from_date(raw_trade['time'], 'iso8601', 'FTX')
trade_type = deserialize_trade_type(raw_trade['side'])
base_asset = asset_from_ftx(raw_trade['baseCurrency'])
quote_asset = asset_from_ftx(raw_trade['quoteCurrency'])
amount = deserialize_asset_amount(raw_trade['size'])
rate = deserialize_price(raw_trade['price'])
fee_currency = asset_from_ftx(raw_trade['feeCurrency'])
fee = deserialize_fee(raw_trade['fee'])
return Trade(
timestamp=timestamp,
location=Location.FTX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(raw_trade['id']),
)
def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the fee_currency string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
pair = data['pair']
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = deserialize_trade_type(data['trade_type'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=data['location'],
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
As we saw in https://github.com/rotki/rotki/issues/1281 it's quite possible
that some keys don't exist in a trade. The required fields are here:
https://bittrex.github.io/api/v3#definition-Order
May raise:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = deserialize_asset_amount(bittrex_trade['fillQuantity'])
timestamp = deserialize_timestamp_from_date(
date=bittrex_trade['closedAt'], # we only check closed orders
formatstr='iso8601',
location='bittrex',
)
if 'limit' in bittrex_trade:
rate = deserialize_price(bittrex_trade['limit'])
else:
rate = Price(
deserialize_asset_amount(bittrex_trade['proceeds']) /
deserialize_asset_amount(bittrex_trade['fillQuantity']),
)
order_type = deserialize_trade_type(bittrex_trade['direction'])
fee = deserialize_fee(bittrex_trade['commission'])
base_asset, quote_asset = bittrex_pair_to_world(bittrex_trade['marketSymbol'])
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
fee=fee,
bittrex_pair=bittrex_trade['marketSymbol'],
base_asset=base_asset,
quote_asset=quote_asset,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_asset,
link=str(bittrex_trade['id']),
)
def trade_from_conversion(trade_a: Dict[str, Any], trade_b: Dict[str, Any]) -> Optional[Trade]:
"""Turn information from a conversion into a trade
Mary raise:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
# Check that the status is complete
if trade_a['status'] != 'completed':
return None
# Trade b will represent the asset we are converting to
if trade_b['amount']['amount'].startswith('-'):
trade_a, trade_b = trade_b, trade_a
timestamp = deserialize_timestamp_from_date(trade_a['updated_at'], 'iso8601', 'coinbase')
trade_type = deserialize_trade_type('sell')
tx_amount = AssetAmount(abs(deserialize_asset_amount(trade_a['amount']['amount'])))
tx_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
native_amount = deserialize_asset_amount(trade_b['amount']['amount'])
native_asset = asset_from_coinbase(trade_b['amount']['currency'], time=timestamp)
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
# Obtain fee amount in the native currency using data from both trades
amount_after_fee = deserialize_asset_amount(trade_b['native_amount']['amount'])
amount_before_fee = deserialize_asset_amount(trade_a['native_amount']['amount'])
# amount_after_fee + amount_before_fee is a negative amount and the fee needs to be positive
conversion_native_fee_amount = abs(amount_after_fee + amount_before_fee)
if ZERO not in (tx_amount, conversion_native_fee_amount, amount_before_fee):
# We have the fee amount in the native currency. To get it in the
# converted asset we have to get the rate
asset_native_rate = tx_amount / abs(amount_before_fee)
fee_amount = Fee(conversion_native_fee_amount / asset_native_rate)
else:
fee_amount = Fee(ZERO)
fee_asset = asset_from_coinbase(trade_a['amount']['currency'], time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
# in coinbase you are buying/selling tx_asset for native_asset
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(trade_a['trade']['id']),
)
def _deserialize(
self,
value: str,
attr, # pylint: disable=unused-argument
data, # pylint: disable=unused-argument
**kwargs, # pylint: disable=unused-argument
) -> TradeType:
try:
trade_type = deserialize_trade_type(value)
except DeserializationError as e:
raise ValidationError(str(e))
return trade_type
def _deserialize(
self,
value: str,
attr: Optional[str], # pylint: disable=unused-argument
data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument
**_kwargs: Any,
) -> TradeType:
try:
trade_type = deserialize_trade_type(value)
except DeserializationError as e:
raise ValidationError(str(e))
return trade_type
def _read_trades(self, filepath: str) -> List[Trade]:
"""Reads a csv fill type report and extracts the Trades"""
with open(filepath, newline='') as csvfile:
reader = csv.DictReader(csvfile)
trades = []
for row in reader:
try:
timestamp = deserialize_timestamp_from_date(
row['created at'],
'iso8601',
'coinbasepro',
)
trades.append(Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
pair=coinbasepro_to_worldpair(row['product']),
trade_type=deserialize_trade_type(row['side']),
amount=deserialize_asset_amount(row['size']),
rate=deserialize_price(row['price']),
fee=deserialize_fee(row['fee']),
fee_currency=Asset(row['price/fee/total unit']),
link=row['trade id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.',
)
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.',
)
log.error(
'Error processing a coinbasepro trade.',
raw_trade=row,
error=msg,
)
continue
return trades
def trade_from_coinbase(raw_trade: Dict[str, Any]) -> Optional[Trade]:
"""Turns a coinbase transaction into a rotkehlchen Trade.
https://developers.coinbase.com/api/v2?python#buys
If the coinbase transaction is not a trade related transaction returns None
Throws:
- UnknownAsset due to Asset instantiation
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entires missing an expected entry
"""
if raw_trade['status'] != 'completed':
# We only want to deal with completed trades
return None
if raw_trade['instant']:
raw_time = raw_trade['created_at']
else:
raw_time = raw_trade['payout_at']
timestamp = deserialize_timestamp_from_date(raw_time, 'iso8601',
'coinbase')
trade_type = deserialize_trade_type(raw_trade['resource'])
tx_amount = deserialize_asset_amount(raw_trade['amount']['amount'])
tx_asset = asset_from_coinbase(raw_trade['amount']['currency'],
time=timestamp)
native_amount = deserialize_asset_amount(raw_trade['subtotal']['amount'])
native_asset = asset_from_coinbase(raw_trade['subtotal']['currency'],
time=timestamp)
# in coinbase you are buying/selling tx_asset for native_asset
pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}')
amount = tx_amount
# The rate is how much you get/give in quotecurrency if you buy/sell 1 unit of base currency
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee']['amount'])
fee_asset = asset_from_coinbase(raw_trade['fee']['currency'],
time=timestamp)
return Trade(
timestamp=timestamp,
location=Location.COINBASE,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['id']),
)
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
if cost != amount * rate:
log.warning(
'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
location='kraken',
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
)
def trade_from_bitcoinde(raw_trade: Dict) -> Trade:
"""Convert bitcoin.de raw data to a trade
May raise:
- DeserializationError
- UnknownAsset
- KeyError
"""
try:
timestamp = deserialize_timestamp_from_date(
raw_trade['successfully_finished_at'],
'iso8601',
'bitcoinde',
)
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = deserialize_timestamp_from_date(
raw_trade['trade_marked_as_paid_at'],
'iso8601',
'bitcoinde',
)
trade_type = deserialize_trade_type(raw_trade['type'])
tx_amount = deserialize_asset_amount(raw_trade['amount_currency_to_trade'])
native_amount = deserialize_asset_amount(
raw_trade['volume_currency_to_pay'])
tx_asset, native_asset = bitcoinde_pair_to_world(raw_trade['trading_pair'])
amount = tx_amount
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee_currency_to_pay'])
fee_asset = Asset('EUR')
return Trade(
timestamp=timestamp,
location=Location.BITCOINDE,
base_asset=tx_asset,
quote_asset=native_asset,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['trade_id']),
)
def trade_from_bittrex(bittrex_trade: Dict[str, Any]) -> Trade:
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format
Throws:
- UnknownAsset/UnsupportedAsset due to bittrex_pair_to_world()
- DeserializationError due to unexpected format of dict entries
- KeyError due to dict entries missing an expected entry
"""
amount = AssetAmount(
deserialize_asset_amount(bittrex_trade['Quantity']) -
deserialize_asset_amount(bittrex_trade['QuantityRemaining']), )
timestamp = deserialize_timestamp_from_bittrex_date(
bittrex_trade['TimeStamp'])
rate = deserialize_price(bittrex_trade['PricePerUnit'])
order_type = deserialize_trade_type(bittrex_trade['OrderType'])
bittrex_price = deserialize_price(bittrex_trade['Price'])
fee = deserialize_fee(bittrex_trade['Commission'])
pair = bittrex_pair_to_world(bittrex_trade['Exchange'])
quote_currency = get_pair_position_asset(pair, 'second')
log.debug(
'Processing bittrex Trade',
sensitive_log=True,
amount=amount,
rate=rate,
order_type=order_type,
price=bittrex_price,
fee=fee,
bittrex_pair=bittrex_trade['Exchange'],
pair=pair,
)
return Trade(
timestamp=timestamp,
location=Location.BITTREX,
pair=pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=str(bittrex_trade['OrderUuid']),
)
def trade_from_bitcoinde(raw_trade: Dict) -> Trade:
try:
timestamp = deserialize_timestamp_from_date(
raw_trade['successfully_finished_at'],
'iso8601',
'bitcoinde',
)
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = deserialize_timestamp_from_date(
raw_trade['trade_marked_as_paid_at'],
'iso8601',
'bitcoinde',
)
trade_type = deserialize_trade_type(raw_trade['type'])
tx_amount = AssetAmount(FVal(raw_trade['amount_currency_to_trade']))
native_amount = FVal(raw_trade['volume_currency_to_pay'])
tx_asset, native_asset = bitcoinde_pair_to_world(raw_trade['trading_pair'])
pair = TradePair(f'{tx_asset.identifier}_{native_asset.identifier}')
amount = tx_amount
rate = Price(native_amount / tx_amount)
fee_amount = deserialize_fee(raw_trade['fee_currency_to_pay'])
fee_asset = Asset('EUR')
return Trade(
timestamp=timestamp,
location=Location.BITCOINDE,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee_amount,
fee_currency=fee_asset,
link=str(raw_trade['trade_id']),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
# first get all orders, to see which product ids we need to query fills for
orders = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
orders.extend(batch)
queried_product_ids = set()
for order_entry in orders:
product_id = order_entry.get('product_id', None)
if product_id is None:
msg = (
'Skipping coinbasepro trade since it lacks a product_id. '
'Check logs for details')
self.msg_aggregator.add_error(msg)
log.error(
'Error processing a coinbasepro order.',
raw_trade=order_entry,
error=msg,
)
continue
if product_id in queried_product_ids:
continue # already queried this product id
# Now let's get all fills for this product id
queried_product_ids.add(product_id)
fills = []
for batch in self._paginated_query(
endpoint='fills',
query_options={'product_id': product_id},
):
fills.extend(batch)
try:
base_asset, quote_asset = coinbasepro_to_worldpair(product_id)
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
for fill_entry in fills:
try:
timestamp = coinbasepro_deserialize_timestamp(
fill_entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
base_asset=base_asset,
quote_asset=quote_asset,
trade_type=deserialize_trade_type(
fill_entry['side']),
amount=deserialize_asset_amount(
fill_entry['size']),
rate=deserialize_price(fill_entry['price']),
fee=deserialize_fee(fill_entry['fee']),
fee_currency=quote_asset,
link=str(fill_entry['trade_id']) + '_' +
fill_entry['order_id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro fill.',
raw_trade=fill_entry,
error=msg,
)
continue
return trades
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format
- Can raise UnknownAsset due to kraken_to_world_pair
- Can raise UnprocessableTradePair due to kraken_to_world_pair
- Can raise DeserializationError due to dict entries not being as expected
- Can raise KeyError due to dict entries missing an expected entry
"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position_asset(currency_pair, 'second')
timestamp = deserialize_timestamp_from_kraken(kraken_trade['time'])
amount = deserialize_asset_amount(kraken_trade['vol'])
cost = deserialize_price(kraken_trade['cost'])
fee = deserialize_fee(kraken_trade['fee'])
order_type = deserialize_trade_type(kraken_trade['type'])
rate = deserialize_price(kraken_trade['price'])
# pylint does not seem to see that Price is essentially FVal
if not cost.is_close(amount * rate): # pylint: disable=no-member
log.warning(
f'cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade'
)
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
# Kraken trades can have the same ordertxid and postxid for different trades ..
# Also note postxid is optional and can be missing
# The only thing that could differentiate them is timestamps in the milliseconds range
# For example here are parts of two different kraken_trade:
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.2955)
# {'ordertxid': 'AM4ZOZ-GLEMD-ZICOGR', 'postxid': 'AKH2SE-M7IF5-CFI7AT',
# 'pair': 'XXBTZEUR', 'time': FVal(1561161486.3005)
#
# In order to counter this for the unique exchange trade link we are going
# to use a concatenation of the above
exchange_uuid = (
str(kraken_trade['ordertxid']) +
str(kraken_trade.get('postxid', '')) + # postxid is optional
str(kraken_trade['time']))
return Trade(
timestamp=timestamp,
location=Location.KRAKEN,
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
link=exchange_uuid,
)
def trade_from_poloniex(poloniex_trade: Dict[str, Any], pair: TradePair) -> Trade:
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format
Throws:
- UnsupportedAsset due to asset_from_poloniex()
- DeserializationError due to the data being in unexpected format
- UnprocessableTradePair due to the pair data being in an unexpected format
"""
try:
trade_type = deserialize_trade_type(poloniex_trade['type'])
amount = deserialize_asset_amount(poloniex_trade['amount'])
rate = deserialize_price(poloniex_trade['rate'])
perc_fee = deserialize_fee(poloniex_trade['fee'])
base_currency = asset_from_poloniex(get_pair_position_str(pair, 'first'))
quote_currency = asset_from_poloniex(get_pair_position_str(pair, 'second'))
timestamp = deserialize_timestamp_from_poloniex_date(poloniex_trade['date'])
except KeyError as e:
raise DeserializationError(
f'Poloniex trade deserialization error. Missing key entry for {str(e)} in trade dict',
)
cost = rate * amount
if trade_type == TradeType.BUY:
fee = Fee(amount * perc_fee)
fee_currency = quote_currency
elif trade_type == TradeType.SELL:
fee = Fee(cost * perc_fee)
fee_currency = base_currency
else:
raise DeserializationError(f'Got unexpected trade type "{trade_type}" for poloniex trade')
if poloniex_trade['category'] == 'settlement':
if trade_type == TradeType.BUY:
trade_type = TradeType.SETTLEMENT_BUY
else:
trade_type = TradeType.SETTLEMENT_SELL
log.debug(
'Processing poloniex Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=trade_type,
pair=pair,
base_currency=base_currency,
quote_currency=quote_currency,
amount=amount,
fee=fee,
rate=rate,
)
# Use the converted assets in our pair
pair = trade_pair_from_assets(base_currency, quote_currency)
# Since in Poloniex the base currency is the cost currency, iow in poloniex
# for BTC_ETH we buy ETH with BTC and sell ETH for BTC, we need to turn it
# into the Rotkehlchen way which is following the base/quote approach.
pair = invert_pair(pair)
return Trade(
timestamp=timestamp,
location=Location.POLONIEX,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(poloniex_trade['globalTradeID']),
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
trades.append(
Trade(
timestamp=timestamp,
location=Location.GEMINI,
pair=gemini_symbol_to_pair(symbol),
trade_type=deserialize_trade_type(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=Asset(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries coinbase pro for trades"""
log.debug('Query coinbasepro trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
raw_trades = []
for batch in self._paginated_query(
endpoint='orders',
query_options={'status': 'done'},
):
raw_trades.extend(batch)
for entry in raw_trades:
timestamp = coinbasepro_deserialize_timestamp(entry, 'created_at')
if timestamp < start_ts or timestamp > end_ts:
continue
try:
pair = coinbasepro_to_worldpair(entry['product_id'])
# Fee currency seems to always be quote asset
# https://github.com/ccxt/ccxt/blob/ddf3a15cbff01541f0b37c35891aa143bb7f9d7b/python/ccxt/coinbasepro.py#L724 # noqa: E501
_, quote_asset = pair_get_assets(pair)
trades.append(
Trade(
timestamp=timestamp,
location=Location.COINBASEPRO,
pair=coinbasepro_to_worldpair(entry['product_id']),
trade_type=deserialize_trade_type(entry['side']),
amount=deserialize_asset_amount(entry['size']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fill_fees']),
fee_currency=quote_asset,
link=entry['id'],
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Coinbasepro pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Coinbasepro asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a coinbasepro trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a coinbasepro trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
# Not catching DeserializationError here since we have asserts for the data
# being strings right above
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = create_timestamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location=Location.EXTERNAL,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''
