def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
page = 1
resp_trades = []
while True:
resp = self._api_query('get', 'trades', {'state': 1, 'page': page})
resp_trades.extend(resp['trades'])
if 'page' not in resp:
break
if resp['page']['current'] >= resp['page']['last']:
break
page = resp['page']['current'] + 1
log.debug('Bitcoin.de trade history query',
results_num=len(resp_trades))
trades = []
for tx in resp_trades:
try:
timestamp = iso8601ts_to_timestamp(
tx['successfully_finished_at'])
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = iso8601ts_to_timestamp(
tx['trade_marked_as_paid_at'])
if tx['state'] != 1:
continue
if timestamp < start_ts or timestamp > end_ts:
continue
try:
trades.append(trade_from_bitcoinde(tx))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitcoin.de trade with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Error processing a Bitcoin.de trade. Check logs '
'for details. Ignoring it.', )
log.error(
'Error processing a Bitcoin.de trade',
trade=tx,
error=msg,
)
continue
return trades
def deserialize_timestamp_from_date(date: str, formatstr: str,
location: str) -> Timestamp:
"""Deserializes a timestamp from a date entry depending on the format str
formatstr can also have a special value of 'iso8601' in which case the iso8601
function will be used.
Can throw DeserializationError if the data is not as expected
"""
if not date:
raise DeserializationError(
f'Failed to deserialize a timestamp from a null entry in {location}',
)
if not isinstance(date, str):
raise DeserializationError(
f'Failed to deserialize a timestamp from a {type(date)} entry in {location}',
)
if formatstr == 'iso8601':
return iso8601ts_to_timestamp(date)
try:
return Timestamp(create_timestamp(datestr=date, formatstr=formatstr))
except ValueError:
raise DeserializationError(
f'Failed to deserialize {date} {location} timestamp entry')
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition:
"""Turn a bitmex trade returned from bitmex trade history to our common trade
history format. This only returns margin positions as bitmex only deals in
margin trading"""
close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime'])
profit_loss = AssetAmount(satoshis_to_btc(FVal(bitmex_trade['amount'])))
currency = bitmex_to_world(bitmex_trade['currency'])
fee = deserialize_fee(bitmex_trade['fee'])
notes = bitmex_trade['address']
assert currency == A_BTC, 'Bitmex trade should only deal in BTC'
log.debug(
'Processing Bitmex Trade',
sensitive_log=True,
timestamp=close_time,
profit_loss=profit_loss,
currency=currency,
fee=fee,
notes=notes,
)
return MarginPosition(
location=Location.BITMEX,
open_time=None,
close_time=close_time,
profit_loss=profit_loss,
pl_currency=currency,
fee=fee,
fee_currency=A_BTC,
notes=notes,
link=str(bitmex_trade['transactID']),
)
def query_online_margin_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[MarginPosition]:
# We know user/walletHistory returns a list
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex trade history query', results_num=len(resp))
margin_trades = []
for tx in resp:
if tx['timestamp'] is None:
timestamp = None
else:
timestamp = iso8601ts_to_timestamp(tx['timestamp'])
if tx['transactType'] != 'RealisedPNL':
continue
if timestamp and timestamp < start_ts:
continue
if timestamp and timestamp > end_ts:
continue
margin_trades.append(trade_from_bitmex(tx))
return margin_trades
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition:
"""Turn a bitmex trade returned from bitmex trade history to our common trade
history format. This only returns margin positions as bitmex only deals in
margin trading"""
close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime'])
profit_loss = satoshis_to_btc(FVal(bitmex_trade['amount']))
currency = bitmex_to_world(bitmex_trade['currency'])
notes = bitmex_trade['address']
assert currency == 'BTC', 'Bitmex trade should only deal in BTC'
log.debug(
'Processing Bitmex Trade',
sensitive_log=True,
timestamp=close_time,
profit_loss=profit_loss,
currency=currency,
notes=notes,
)
return MarginPosition(
exchange='bitmex',
open_time=None,
close_time=close_time,
profit_loss=profit_loss,
pl_currency=A_BTC,
notes=notes,
)
def query_trade_history(
self,
start_ts: typing.Timestamp,
end_ts: typing.Timestamp,
end_at_least_ts: typing.Timestamp, # pylint: disable=unused-argument
market: Optional[str] = None, # pylint: disable=unused-argument
count: Optional[int] = None, # pylint: disable=unused-argument
) -> List:
try:
# We know user/walletHistory returns a list
resp = self._api_query('get', 'user/walletHistory')
except RemoteError as e:
msg = ('Bitmex API request failed. Could not reach bitmex due '
'to {}'.format(e))
log.error(msg)
return list()
log.debug('Bitmex trade history query', results_num=len(resp))
realised_pnls = []
for tx in resp:
if tx['timestamp'] is None:
timestamp = None
else:
timestamp = iso8601ts_to_timestamp(tx['timestamp'])
if tx['transactType'] != 'RealisedPNL':
continue
if timestamp and timestamp < start_ts:
continue
if timestamp and timestamp > end_ts:
continue
realised_pnls.append(tx)
return realised_pnls
def query_deposits_withdrawals(
self,
start_ts: typing.Timestamp,
end_ts: typing.Timestamp,
end_at_least_ts: typing.Timestamp,
) -> List:
# TODO: Implement cache like in other exchange calls
try:
resp = self._api_query_list('get', 'user/walletHistory')
except RemoteError as e:
msg = (
'Bitmex API request failed. Could not reach bitmex due '
'to {}'.format(e)
)
log.error(msg)
return list()
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
transaction_type = movement['transactType']
if transaction_type not in ('Deposit', 'Withdrawal'):
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = FVal(movement['amount'])
fee = ZERO
if movement['fee'] is not None:
fee = FVal(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == 'BTC':
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(AssetMovement(
exchange='bitmex',
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=Fee(fee),
))
return movements
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
page = 1
resp_trades = []
while True:
resp = self._api_query('get', 'trades', {'state': 1, 'page': page})
resp_trades.extend(resp['trades'])
if 'page' not in resp:
break
if resp['page']['current'] >= resp['page']['last']:
break
page = resp['page']['current'] + 1
log.debug('Bitcoin.de trade history query',
results_num=len(resp_trades))
trades = []
for tx in resp_trades:
try:
timestamp = iso8601ts_to_timestamp(
tx['successfully_finished_at'])
except KeyError:
# For very old trades (2013) bitcoin.de does not return 'successfully_finished_at'
timestamp = iso8601ts_to_timestamp(
tx['trade_marked_as_paid_at'])
if tx['state'] != 1:
continue
if timestamp < start_ts or timestamp > end_ts:
continue
trades.append(trade_from_bitcoinde(tx))
return trades
def deserialize_timestamp_from_date(
date: Optional[str],
formatstr: str,
location: str,
skip_milliseconds: bool = False,
) -> Timestamp:
"""Deserializes a timestamp from a date entry depending on the format str
formatstr can also have a special value of 'iso8601' in which case the iso8601
function will be used.
Can throw DeserializationError if the data is not as expected
"""
if not date:
raise DeserializationError(
f'Failed to deserialize a timestamp from a null entry in {location}',
)
if not isinstance(date, str):
raise DeserializationError(
f'Failed to deserialize a timestamp from a {type(date)} entry in {location}',
)
if skip_milliseconds:
# Seems that poloniex added milliseconds in their timestamps.
# https://github.com/rotki/rotki/issues/1631
# We don't deal with milliseconds in rotki times so we can safely remove it
splits = date.split('.', 1)
if len(splits) == 2:
date = splits[0]
if formatstr == 'iso8601':
return iso8601ts_to_timestamp(date)
date = date.rstrip('Z')
try:
return Timestamp(create_timestamp(datestr=date, formatstr=formatstr))
except ValueError as e:
raise DeserializationError(
f'Failed to deserialize {date} {location} timestamp entry',
) from e
def timerange_check(
asset_symbol: str,
our_asset: Dict[str, Any],
our_data: Dict[str, Any],
paprika_data: Dict[str, Any],
cmc_data: Dict[str, Any],
always_keep_our_time: bool,
token_address: EthAddress = None,
) -> Dict[str, Any]:
"""Process the started timestamps from coin paprika and coinmarketcap.
Then compare to our data and provide choices to clean up the data.
"""
if Asset(asset_symbol).is_fiat():
# Fiat does not have started date (or we don't care about it)
return our_data
paprika_started = None
if paprika_data:
paprika_started = paprika_data['started_at']
cmc_started = None
if cmc_data:
cmc_started = cmc_data['first_historical_data']
if not cmc_started and not paprika_started and not token_address:
print(f'Did not find a started date for asset {asset_symbol} in any of the external APIs')
return our_data
paprika_started_ts = None
if paprika_started:
paprika_started_ts = create_timestamp(paprika_started, formatstr='%Y-%m-%dT%H:%M:%SZ')
cmc_started_ts = None
if cmc_data:
cmc_started_ts = iso8601ts_to_timestamp(cmc_started)
if asset_symbol in PREFER_OUR_STARTED:
assert 'started' in our_asset
# Already manually checked
return our_data
our_started = our_asset.get('started', None)
# if it's an eth token entry, get the contract creation time too
if token_address:
contract_creation_ts = get_token_contract_creation_time(token_address)
if not our_started:
# If we don't have any data and CMC and paprika agree just use their timestamp
if cmc_started == paprika_started and cmc_started is not None:
our_data[asset_symbol]['started'] = cmc_started
return our_data
if our_started and always_keep_our_time:
return our_data
if our_started is None or our_started != cmc_started or our_started != paprika_started:
choices = (1, 2, 3)
msg = (
f'For asset {asset_symbol} the started times are: \n'
f'(1) Our data: {our_started} -- {timestamp_to_date(our_started) if our_started else ""}\n'
f'(2) Coinpaprika: {paprika_started_ts} -- '
f'{timestamp_to_date(paprika_started_ts) if paprika_started_ts else ""}\n'
f'(3) Coinmarketcap: {cmc_started_ts} -- '
f'{timestamp_to_date(cmc_started_ts) if cmc_started_ts else ""} \n'
)
if token_address:
msg += (
f'(4) Contract creation: {contract_creation_ts} -- '
f'{timestamp_to_date(contract_creation_ts) if contract_creation_ts else ""}\n'
)
choices = (1, 2, 3, 4)
msg += f'Choose a number (1)-({choices[-1]}) to choose which timestamp to use: '
choice = choose_multiple(msg, choices)
if choice == 1:
if not our_started:
print('Chose our timestamp but we got no timestamp. Bailing ...')
sys.exit(1)
timestamp = our_started
elif choice == 2:
if not paprika_started_ts:
print("Chose coin paprika's timestamp but it's empty. Bailing ...")
sys.exit(1)
timestamp = paprika_started_ts
elif choice == 3:
if not cmc_started_ts:
print("Chose coinmarketcap's timestamp but it's empty. Bailing ...")
sys.exit(1)
timestamp = cmc_started_ts
elif choice == 4:
if not contract_creation_ts:
print("Chose contract creation timestamp but it's empty. Bailing ...")
sys.exit(1)
timestamp = contract_creation_ts
our_data[asset_symbol]['started'] = timestamp
return our_data
def test_iso8601ts_to_timestamp():
assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400
assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202
assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505438
# Timezone specific part of the test
timezone_ts_str = '1997-07-16T22:30'
timezone_ts_at_utc = 869092200
assert iso8601ts_to_timestamp(timezone_ts_str + 'Z') == timezone_ts_at_utc
# The utc offset for July should be time.altzone since it's in DST
# https://stackoverflow.com/questions/3168096/getting-computers-utc-offset-in-python
utc_offset = time.altzone
assert iso8601ts_to_timestamp(
timezone_ts_str) == timezone_ts_at_utc + utc_offset
assert iso8601ts_to_timestamp('1997-07-16T22:30+01:00') == 869088600
assert iso8601ts_to_timestamp('1997-07-16T22:30:45+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.1+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.01+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.001+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.9+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.99+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.999+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T21:30:45+00:00') == 869088645
def query_online_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List:
resp = self._api_query_list('get', 'user/walletHistory')
log.debug('Bitmex deposit/withdrawals query', results_num=len(resp))
movements = list()
for movement in resp:
try:
transaction_type = movement['transactType']
if transaction_type == 'Deposit':
transaction_type = AssetMovementCategory.DEPOSIT
elif transaction_type == 'Withdrawal':
transaction_type = AssetMovementCategory.WITHDRAWAL
else:
continue
timestamp = iso8601ts_to_timestamp(movement['timestamp'])
if timestamp < start_ts:
continue
if timestamp > end_ts:
continue
asset = bitmex_to_world(movement['currency'])
amount = deserialize_asset_amount(movement['amount'])
fee = deserialize_fee(movement['fee'])
# bitmex has negative numbers for withdrawals
if amount < 0:
amount *= -1
if asset == A_BTC:
# bitmex stores amounts in satoshis
amount = satoshis_to_btc(amount)
fee = satoshis_to_btc(fee)
movements.append(
AssetMovement(
location=Location.BITMEX,
category=transaction_type,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=fee,
link=str(movement['transactID']),
))
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found bitmex deposit/withdrawal with unknown asset '
f'{e.asset_name}. Ignoring it.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
f'Unexpected data encountered during deserialization of a bitmex '
f'asset movement. Check logs for details and open a bug report.',
)
log.error(
f'Unexpected data encountered during deserialization of bitmex '
f'asset_movement {movement}. Error was: {str(e)}', )
continue
return movements
def test_iso8601ts_to_timestamp():
assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400
assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202
assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505437
def test_iso8601ts_to_timestamp():
# Get timezone offset to UTC
utc_offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone
assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400
assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202
assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505438
# Timezone specific part of the test
timezone_ts_str = '1997-07-16T22:30'
timezone_ts_at_utc = 869092200
assert iso8601ts_to_timestamp(timezone_ts_str + 'Z') == timezone_ts_at_utc
assert iso8601ts_to_timestamp(timezone_ts_str) == timezone_ts_at_utc + utc_offset
assert iso8601ts_to_timestamp('1997-07-16T22:30+01:00') == 869088600
assert iso8601ts_to_timestamp('1997-07-16T22:30:45+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.1+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.01+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.001+01:00') == 869088645
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.9+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.99+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T22:30:45.999+01:00') == 869088646
assert iso8601ts_to_timestamp('1997-07-16T21:30:45+00:00') == 869088645
