def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: page = 1 resp_trades = [] while True: resp = self._api_query('get', 'trades', {'state': 1, 'page': page}) resp_trades.extend(resp['trades']) if 'page' not in resp: break if resp['page']['current'] >= resp['page']['last']: break page = resp['page']['current'] + 1 log.debug('Bitcoin.de trade history query', results_num=len(resp_trades)) trades = [] for tx in resp_trades: try: timestamp = iso8601ts_to_timestamp( tx['successfully_finished_at']) except KeyError: # For very old trades (2013) bitcoin.de does not return 'successfully_finished_at' timestamp = iso8601ts_to_timestamp( tx['trade_marked_as_paid_at']) if tx['state'] != 1: continue if timestamp < start_ts or timestamp > end_ts: continue try: trades.append(trade_from_bitcoinde(tx)) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bitcoin.de trade with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( 'Error processing a Bitcoin.de trade. Check logs ' 'for details. Ignoring it.', ) log.error( 'Error processing a Bitcoin.de trade', trade=tx, error=msg, ) continue return trades
def deserialize_timestamp_from_date(date: str, formatstr: str, location: str) -> Timestamp: """Deserializes a timestamp from a date entry depending on the format str formatstr can also have a special value of 'iso8601' in which case the iso8601 function will be used. Can throw DeserializationError if the data is not as expected """ if not date: raise DeserializationError( f'Failed to deserialize a timestamp from a null entry in {location}', ) if not isinstance(date, str): raise DeserializationError( f'Failed to deserialize a timestamp from a {type(date)} entry in {location}', ) if formatstr == 'iso8601': return iso8601ts_to_timestamp(date) try: return Timestamp(create_timestamp(datestr=date, formatstr=formatstr)) except ValueError: raise DeserializationError( f'Failed to deserialize {date} {location} timestamp entry')
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition: """Turn a bitmex trade returned from bitmex trade history to our common trade history format. This only returns margin positions as bitmex only deals in margin trading""" close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime']) profit_loss = AssetAmount(satoshis_to_btc(FVal(bitmex_trade['amount']))) currency = bitmex_to_world(bitmex_trade['currency']) fee = deserialize_fee(bitmex_trade['fee']) notes = bitmex_trade['address'] assert currency == A_BTC, 'Bitmex trade should only deal in BTC' log.debug( 'Processing Bitmex Trade', sensitive_log=True, timestamp=close_time, profit_loss=profit_loss, currency=currency, fee=fee, notes=notes, ) return MarginPosition( location=Location.BITMEX, open_time=None, close_time=close_time, profit_loss=profit_loss, pl_currency=currency, fee=fee, fee_currency=A_BTC, notes=notes, link=str(bitmex_trade['transactID']), )
def query_online_margin_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[MarginPosition]: # We know user/walletHistory returns a list resp = self._api_query_list('get', 'user/walletHistory') log.debug('Bitmex trade history query', results_num=len(resp)) margin_trades = [] for tx in resp: if tx['timestamp'] is None: timestamp = None else: timestamp = iso8601ts_to_timestamp(tx['timestamp']) if tx['transactType'] != 'RealisedPNL': continue if timestamp and timestamp < start_ts: continue if timestamp and timestamp > end_ts: continue margin_trades.append(trade_from_bitmex(tx)) return margin_trades
def trade_from_bitmex(bitmex_trade: Dict) -> MarginPosition: """Turn a bitmex trade returned from bitmex trade history to our common trade history format. This only returns margin positions as bitmex only deals in margin trading""" close_time = iso8601ts_to_timestamp(bitmex_trade['transactTime']) profit_loss = satoshis_to_btc(FVal(bitmex_trade['amount'])) currency = bitmex_to_world(bitmex_trade['currency']) notes = bitmex_trade['address'] assert currency == 'BTC', 'Bitmex trade should only deal in BTC' log.debug( 'Processing Bitmex Trade', sensitive_log=True, timestamp=close_time, profit_loss=profit_loss, currency=currency, notes=notes, ) return MarginPosition( exchange='bitmex', open_time=None, close_time=close_time, profit_loss=profit_loss, pl_currency=A_BTC, notes=notes, )
def query_trade_history( self, start_ts: typing.Timestamp, end_ts: typing.Timestamp, end_at_least_ts: typing.Timestamp, # pylint: disable=unused-argument market: Optional[str] = None, # pylint: disable=unused-argument count: Optional[int] = None, # pylint: disable=unused-argument ) -> List: try: # We know user/walletHistory returns a list resp = self._api_query('get', 'user/walletHistory') except RemoteError as e: msg = ('Bitmex API request failed. Could not reach bitmex due ' 'to {}'.format(e)) log.error(msg) return list() log.debug('Bitmex trade history query', results_num=len(resp)) realised_pnls = [] for tx in resp: if tx['timestamp'] is None: timestamp = None else: timestamp = iso8601ts_to_timestamp(tx['timestamp']) if tx['transactType'] != 'RealisedPNL': continue if timestamp and timestamp < start_ts: continue if timestamp and timestamp > end_ts: continue realised_pnls.append(tx) return realised_pnls
def query_deposits_withdrawals( self, start_ts: typing.Timestamp, end_ts: typing.Timestamp, end_at_least_ts: typing.Timestamp, ) -> List: # TODO: Implement cache like in other exchange calls try: resp = self._api_query_list('get', 'user/walletHistory') except RemoteError as e: msg = ( 'Bitmex API request failed. Could not reach bitmex due ' 'to {}'.format(e) ) log.error(msg) return list() log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: transaction_type = movement['transactType'] if transaction_type not in ('Deposit', 'Withdrawal'): continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = FVal(movement['amount']) fee = ZERO if movement['fee'] is not None: fee = FVal(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == 'BTC': # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append(AssetMovement( exchange='bitmex', category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee=Fee(fee), )) return movements
def query_online_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: page = 1 resp_trades = [] while True: resp = self._api_query('get', 'trades', {'state': 1, 'page': page}) resp_trades.extend(resp['trades']) if 'page' not in resp: break if resp['page']['current'] >= resp['page']['last']: break page = resp['page']['current'] + 1 log.debug('Bitcoin.de trade history query', results_num=len(resp_trades)) trades = [] for tx in resp_trades: try: timestamp = iso8601ts_to_timestamp( tx['successfully_finished_at']) except KeyError: # For very old trades (2013) bitcoin.de does not return 'successfully_finished_at' timestamp = iso8601ts_to_timestamp( tx['trade_marked_as_paid_at']) if tx['state'] != 1: continue if timestamp < start_ts or timestamp > end_ts: continue trades.append(trade_from_bitcoinde(tx)) return trades
def deserialize_timestamp_from_date( date: Optional[str], formatstr: str, location: str, skip_milliseconds: bool = False, ) -> Timestamp: """Deserializes a timestamp from a date entry depending on the format str formatstr can also have a special value of 'iso8601' in which case the iso8601 function will be used. Can throw DeserializationError if the data is not as expected """ if not date: raise DeserializationError( f'Failed to deserialize a timestamp from a null entry in {location}', ) if not isinstance(date, str): raise DeserializationError( f'Failed to deserialize a timestamp from a {type(date)} entry in {location}', ) if skip_milliseconds: # Seems that poloniex added milliseconds in their timestamps. # https://github.com/rotki/rotki/issues/1631 # We don't deal with milliseconds in rotki times so we can safely remove it splits = date.split('.', 1) if len(splits) == 2: date = splits[0] if formatstr == 'iso8601': return iso8601ts_to_timestamp(date) date = date.rstrip('Z') try: return Timestamp(create_timestamp(datestr=date, formatstr=formatstr)) except ValueError as e: raise DeserializationError( f'Failed to deserialize {date} {location} timestamp entry', ) from e
def timerange_check( asset_symbol: str, our_asset: Dict[str, Any], our_data: Dict[str, Any], paprika_data: Dict[str, Any], cmc_data: Dict[str, Any], always_keep_our_time: bool, token_address: EthAddress = None, ) -> Dict[str, Any]: """Process the started timestamps from coin paprika and coinmarketcap. Then compare to our data and provide choices to clean up the data. """ if Asset(asset_symbol).is_fiat(): # Fiat does not have started date (or we don't care about it) return our_data paprika_started = None if paprika_data: paprika_started = paprika_data['started_at'] cmc_started = None if cmc_data: cmc_started = cmc_data['first_historical_data'] if not cmc_started and not paprika_started and not token_address: print(f'Did not find a started date for asset {asset_symbol} in any of the external APIs') return our_data paprika_started_ts = None if paprika_started: paprika_started_ts = create_timestamp(paprika_started, formatstr='%Y-%m-%dT%H:%M:%SZ') cmc_started_ts = None if cmc_data: cmc_started_ts = iso8601ts_to_timestamp(cmc_started) if asset_symbol in PREFER_OUR_STARTED: assert 'started' in our_asset # Already manually checked return our_data our_started = our_asset.get('started', None) # if it's an eth token entry, get the contract creation time too if token_address: contract_creation_ts = get_token_contract_creation_time(token_address) if not our_started: # If we don't have any data and CMC and paprika agree just use their timestamp if cmc_started == paprika_started and cmc_started is not None: our_data[asset_symbol]['started'] = cmc_started return our_data if our_started and always_keep_our_time: return our_data if our_started is None or our_started != cmc_started or our_started != paprika_started: choices = (1, 2, 3) msg = ( f'For asset {asset_symbol} the started times are: \n' f'(1) Our data: {our_started} -- {timestamp_to_date(our_started) if our_started else ""}\n' f'(2) Coinpaprika: {paprika_started_ts} -- ' f'{timestamp_to_date(paprika_started_ts) if paprika_started_ts else ""}\n' f'(3) Coinmarketcap: {cmc_started_ts} -- ' f'{timestamp_to_date(cmc_started_ts) if cmc_started_ts else ""} \n' ) if token_address: msg += ( f'(4) Contract creation: {contract_creation_ts} -- ' f'{timestamp_to_date(contract_creation_ts) if contract_creation_ts else ""}\n' ) choices = (1, 2, 3, 4) msg += f'Choose a number (1)-({choices[-1]}) to choose which timestamp to use: ' choice = choose_multiple(msg, choices) if choice == 1: if not our_started: print('Chose our timestamp but we got no timestamp. Bailing ...') sys.exit(1) timestamp = our_started elif choice == 2: if not paprika_started_ts: print("Chose coin paprika's timestamp but it's empty. Bailing ...") sys.exit(1) timestamp = paprika_started_ts elif choice == 3: if not cmc_started_ts: print("Chose coinmarketcap's timestamp but it's empty. Bailing ...") sys.exit(1) timestamp = cmc_started_ts elif choice == 4: if not contract_creation_ts: print("Chose contract creation timestamp but it's empty. Bailing ...") sys.exit(1) timestamp = contract_creation_ts our_data[asset_symbol]['started'] = timestamp return our_data
def test_iso8601ts_to_timestamp(): assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400 assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202 assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505438 # Timezone specific part of the test timezone_ts_str = '1997-07-16T22:30' timezone_ts_at_utc = 869092200 assert iso8601ts_to_timestamp(timezone_ts_str + 'Z') == timezone_ts_at_utc # The utc offset for July should be time.altzone since it's in DST # https://stackoverflow.com/questions/3168096/getting-computers-utc-offset-in-python utc_offset = time.altzone assert iso8601ts_to_timestamp( timezone_ts_str) == timezone_ts_at_utc + utc_offset assert iso8601ts_to_timestamp('1997-07-16T22:30+01:00') == 869088600 assert iso8601ts_to_timestamp('1997-07-16T22:30:45+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.1+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.01+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.001+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.9+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.99+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.999+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T21:30:45+00:00') == 869088645
def query_online_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List: resp = self._api_query_list('get', 'user/walletHistory') log.debug('Bitmex deposit/withdrawals query', results_num=len(resp)) movements = list() for movement in resp: try: transaction_type = movement['transactType'] if transaction_type == 'Deposit': transaction_type = AssetMovementCategory.DEPOSIT elif transaction_type == 'Withdrawal': transaction_type = AssetMovementCategory.WITHDRAWAL else: continue timestamp = iso8601ts_to_timestamp(movement['timestamp']) if timestamp < start_ts: continue if timestamp > end_ts: continue asset = bitmex_to_world(movement['currency']) amount = deserialize_asset_amount(movement['amount']) fee = deserialize_fee(movement['fee']) # bitmex has negative numbers for withdrawals if amount < 0: amount *= -1 if asset == A_BTC: # bitmex stores amounts in satoshis amount = satoshis_to_btc(amount) fee = satoshis_to_btc(fee) movements.append( AssetMovement( location=Location.BITMEX, category=transaction_type, timestamp=timestamp, asset=asset, amount=amount, fee_asset=asset, fee=fee, link=str(movement['transactID']), )) except UnknownAsset as e: self.msg_aggregator.add_warning( f'Found bitmex deposit/withdrawal with unknown asset ' f'{e.asset_name}. Ignoring it.', ) continue except (DeserializationError, KeyError) as e: msg = str(e) if isinstance(e, KeyError): msg = f'Missing key entry for {msg}.' self.msg_aggregator.add_error( f'Unexpected data encountered during deserialization of a bitmex ' f'asset movement. Check logs for details and open a bug report.', ) log.error( f'Unexpected data encountered during deserialization of bitmex ' f'asset_movement {movement}. Error was: {str(e)}', ) continue return movements
def test_iso8601ts_to_timestamp(): assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400 assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202 assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505437
def test_iso8601ts_to_timestamp(): # Get timezone offset to UTC utc_offset = time.timezone if (time.localtime().tm_isdst == 0) else time.altzone assert iso8601ts_to_timestamp('2018-09-09T12:00:00.000Z') == 1536494400 assert iso8601ts_to_timestamp('2011-01-01T04:13:22.220Z') == 1293855202 assert iso8601ts_to_timestamp('1986-11-04T16:23:57.921Z') == 531505438 # Timezone specific part of the test timezone_ts_str = '1997-07-16T22:30' timezone_ts_at_utc = 869092200 assert iso8601ts_to_timestamp(timezone_ts_str + 'Z') == timezone_ts_at_utc assert iso8601ts_to_timestamp(timezone_ts_str) == timezone_ts_at_utc + utc_offset assert iso8601ts_to_timestamp('1997-07-16T22:30+01:00') == 869088600 assert iso8601ts_to_timestamp('1997-07-16T22:30:45+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.1+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.01+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.001+01:00') == 869088645 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.9+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.99+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T22:30:45.999+01:00') == 869088646 assert iso8601ts_to_timestamp('1997-07-16T21:30:45+00:00') == 869088645