def query_balances(self) -> ExchangeQueryBalances:
try:
resp = self.api_query_dict('returnCompleteBalances',
{"account": "all"})
except RemoteError as e:
msg = ('Poloniex API request failed. Could not reach poloniex due '
'to {}'.format(e))
log.error(msg)
return None, msg
assets_balance: Dict[Asset, Balance] = {}
for poloniex_asset, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if available != ZERO or on_orders != ZERO:
try:
asset = asset_from_poloniex(poloniex_asset)
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'Found unsupported poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown poloniex asset {e.asset_name}. '
f' Ignoring its balance query.', )
continue
except DeserializationError:
log.error(
f'Unexpected poloniex asset type. Expected string '
f' but got {type(poloniex_asset)}', )
self.msg_aggregator.add_error(
'Found poloniex asset entry with non-string type. '
' Ignoring its balance query.', )
continue
if asset == Asset(
'LEND'): # poloniex mistakenly returns LEND balances
continue # https://github.com/rotki/rotki/issues/2530
try:
usd_price = Inquirer().find_usd_price(asset=asset)
except RemoteError as e:
self.msg_aggregator.add_error(
f'Error processing poloniex balance entry due to inability to '
f'query USD price: {str(e)}. Skipping balance entry', )
continue
amount = available + on_orders
usd_value = amount * usd_price
assets_balance[asset] = Balance(
amount=amount,
usd_value=usd_value,
)
log.debug(
'Poloniex balance query',
sensitive_log=True,
currency=asset,
amount=amount,
usd_value=usd_value,
)
return assets_balance, ''
def test_repr():
btc_repr = repr(Asset('BTC'))
assert btc_repr == '<Asset identifier:BTC name:Bitcoin symbol:BTC>'
def verify_otctrade_data(data: ExternalTrade, ) -> Tuple[Optional[Trade], str]:
"""
Takes in the trade data dictionary, validates it and returns a trade instance
If there is an error it returns an error message in the second part of the tuple
"""
for field in otc_fields:
if field not in data:
return None, f'{field} was not provided'
if data[field] in ('', None) and field not in otc_optional_fields:
return None, f'{field} was empty'
if field in otc_numerical_fields and not is_number(data[field]):
return None, f'{field} should be a number'
# Satisfy mypy typing
assert isinstance(data['otc_pair'], str)
assert isinstance(data['otc_fee_currency'], str)
assert isinstance(data['otc_fee'], str)
pair = TradePair(data['otc_pair'])
try:
first = get_pair_position_asset(pair, 'first')
second = get_pair_position_asset(pair, 'second')
fee_currency = Asset(data['otc_fee_currency'])
except UnknownAsset as e:
return None, f'Provided asset {e.asset_name} is not known to Rotkehlchen'
try:
trade_type = deserialize_trade_type(str(data['otc_type']))
amount = deserialize_asset_amount(data['otc_amount'])
rate = deserialize_price(data['otc_rate'])
fee = deserialize_fee(data['otc_fee'])
except DeserializationError as e:
return None, f'Deserialization Error: {str(e)}'
try:
assert isinstance(data['otc_timestamp'], str)
timestamp = createTimeStamp(data['otc_timestamp'],
formatstr='%d/%m/%Y %H:%M')
except ValueError as e:
return None, f'Could not process the given datetime: {e}'
log.debug(
'Creating OTC trade data',
sensitive_log=True,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
)
if data['otc_fee_currency'] not in (first, second):
return None, 'Trade fee currency should be one of the two in the currency pair'
if data['otc_type'] not in ('buy', 'sell'):
return None, 'Trade type can only be buy or sell'
trade = Trade(
timestamp=timestamp,
location='external',
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link=str(data['otc_link']),
notes=str(data['otc_notes']),
)
return trade, ''
def _scrape_xratescom_exchange_rates(url: str) -> Dict[Asset, Price]:
"""
Scrapes x-rates.com website for the exchange rates tables
May raise:
- RemoteError if we can't query x-rates.com
"""
log.debug(f'Querying x-rates.com stats: {url}')
prices = {}
try:
response = requests.get(url=url, timeout=DEFAULT_TIMEOUT_TUPLE)
except requests.exceptions.RequestException as e:
raise RemoteError(
f'x-rates.com request {url} failed due to {str(e)}') from e
if response.status_code != 200:
raise RemoteError(
f'x-rates.com request {url} failed with code: {response.status_code}'
f' and response: {response.text}', )
soup = BeautifulSoup(
response.text,
'html.parser',
parse_only=SoupStrainer('table', {'class': 'tablesorter ratesTable'}),
)
if soup is None:
raise RemoteError(
'Could not find <table> while parsing x-rates stats page')
try:
tr = soup.table.tbody.tr
except AttributeError as e:
raise RemoteError(
'Could not find first <tr> while parsing x-rates.com page') from e
while tr is not None:
secondtd = tr.select('td:nth-of-type(2)')[0]
try:
href = secondtd.a['href']
except (AttributeError, KeyError) as e:
raise RemoteError(
'Could not find a href of 2nd td while parsing x-rates.com page'
) from e # noqa: E501
parts = href.split('to=')
if len(parts) != 2:
raise RemoteError(
f'Could not find to= in {href} while parsing x-rates.com page')
try:
to_asset = Asset(parts[1])
if not to_asset.is_fiat():
raise ValueError
except (UnknownAsset, ValueError):
log.debug(
f'Skipping {parts[1]} asset because its not a known fiat asset while parsing x-rates.com page'
) # noqa: E501
tr = tr.find_next_sibling()
continue
try:
price = deserialize_price(secondtd.a.text)
except DeserializationError as e:
log.debug(
f'Could not parse x-rates.com rate of {to_asset.identifier} due to {str(e)}. Skipping ...'
) # noqa: E501
tr = tr.find_next_sibling()
continue
prices[to_asset] = price
tr = tr.find_next_sibling()
return prices
def process_asset(
our_data: Dict[str, Dict[str, Any]],
asset_symbol: str,
paprika_coins_list: List[Dict[str, Any]],
paprika: CoinPaprika,
cmc_list: Optional[List[Dict[str, Any]]],
cryptocompare_coins_map: Dict[str, Any],
always_keep_our_time: bool,
) -> Dict[str, Any]:
"""
Process a single asset symbol. Compare to all external APIs and if there is no
local data on the symbol query the user on which data to use for each asset attribute.
"""
token_address = None
our_asset = our_data[asset_symbol]
# Coin paprika does not have info on FIAT currencies
if Asset(asset_symbol).is_fiat():
return our_data
found_coin_id = find_paprika_coin_id(asset_symbol, paprika_coins_list)
if found_coin_id:
print(f'paprika id: {found_coin_id}')
paprika_coin_data = paprika.get_coin_by_id(found_coin_id)
paprika_token_address = get_paprika_data_eth_token_address(
paprika_data=paprika_coin_data,
asset_symbol=asset_symbol,
)
check_paprika_token_address(
paprika_token_address=paprika_token_address,
given_token_address=token_address,
asset_symbol=asset_symbol,
)
else:
paprika_coin_data = None
cmc_coin_data = find_cmc_coin_data(asset_symbol, cmc_list)
our_data = timerange_check(
asset_symbol=asset_symbol,
our_asset=our_asset,
our_data=our_data,
paprika_data=paprika_coin_data,
cmc_data=cmc_coin_data,
always_keep_our_time=always_keep_our_time,
token_address=token_address,
)
our_data = name_check(
asset_symbol=asset_symbol,
our_asset=our_asset,
our_data=our_data,
paprika_data=paprika_coin_data,
cmc_data=cmc_coin_data,
)
our_data = active_check(
asset_symbol=asset_symbol,
our_asset=our_asset,
our_data=our_data,
paprika_data=paprika_coin_data,
cmc_data=cmc_coin_data,
)
our_data = typeinfo_check(
asset_symbol=asset_symbol,
our_asset=our_asset,
our_data=our_data,
paprika_data=paprika_coin_data,
cmc_data=cmc_coin_data,
)
# add the symbol as an asset attribute in the data
symbol = asset_symbol
match = re.search('(.*)-\\d+', symbol)
# If our key is a numbered key, like 'PAI-2', 'PAI-3' e.t.c. use the
# non suffixed symbol, iow just 'PAI'
if match:
symbol = match.group(1)
our_data[asset_symbol]['symbol'] = symbol
if asset_symbol not in cryptocompare_coins_map:
print(f'Asset {asset_symbol} is not in cryptocompare')
sys.exit(1)
return our_data
def test_query_asset_movements_sandbox(
sandbox_kuckoin,
inquirer, # pylint: disable=unused-argument
):
"""Unfortunately the sandbox environment does not support deposits and
withdrawals, therefore they must be mocked.
Below a list of the movements and their timestamps in ascending mode:
Deposits:
- deposit 1 - deposit: 1612556651
- deposit 2 - deposit: 1612556652
- deposit 3 - deposit: 1612556653 -> skipped, inner deposit
Withdrawals:
- withdraw 1: 1612556651 -> skipped, inner withdraw
- withdraw 2: 1612556652
- withdraw 3: 1612556656 -> never requested
By requesting trades from 1612556651 to 1612556654 and patching the time
step as 2s (via MONTHS_IN_SECONDS) we should get back 3 movements.
"""
deposits_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1,
"fee":0.0001,
"currency":"KCS",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5a",
"status":"SUCCESS",
"remark":"movement 2 - deposit",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"address":"0x5f047b29041bcfdbf0e4478cdfa753a336ba6989",
"memo":"5c247c8a03aa677cea2a251d",
"amount":1000,
"fee":0.01,
"currency":"LINK",
"isInner":false,
"walletTxId":"5bbb57386d99522d9f954c5b@test",
"status":"SUCCESS",
"remark":"movement 1 - deposit",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
deposits_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":1,
"fee":0.1,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada6",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 4 - deposit",
"createdAt":1612556653000,
"updatedAt":1612556653000
}
]
}
}
""")
withdrawals_response_1 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":2,
"totalNum":2,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a4",
"address":"1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt",
"memo":"",
"currency":"BCHSV",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 4 - withdraw",
"createdAt":1612556652000,
"updatedAt":1612556652000
},
{
"id":"5c2dc64e03aa675aa263f1a3",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7fe9",
"memo":"",
"currency":"ETH",
"amount":1,
"fee":0.01,
"walletTxId":"3e2414d82acce78d38be7fe9",
"isInner":true,
"status":"SUCCESS",
"remark":"movement 3 - withdraw",
"createdAt":1612556651000,
"updatedAt":1612556651000
}
]
}
}
""")
withdrawals_response_2 = ("""
{
"code":"200000",
"data":{
"currentPage":0,
"pageSize":0,
"totalNum":0,
"totalPage":0,
"items":[]
}
}
""")
withdrawals_response_3 = ("""
{
"code":"200000",
"data":{
"currentPage":1,
"pageSize":1,
"totalNum":1,
"totalPage":1,
"items":[
{
"id":"5c2dc64e03aa675aa263f1a5",
"address":"0x5bedb060b8eb8d823e2414d82acce78d38be7f00",
"memo":"",
"currency":"KCS",
"amount":2.5,
"fee":0.25,
"walletTxId":"b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada5",
"isInner":false,
"status":"SUCCESS",
"remark":"movement 5 - withdraw",
"createdAt":1612556655000,
"updatedAt":1612556655000
}
]
}
}
""")
expected_asset_movements = [
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556652),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5a',
asset=Asset('KCS'),
amount=AssetAmount(FVal('1')),
fee_asset=Asset('KCS'),
fee=Fee(FVal('0.0001')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.DEPOSIT,
timestamp=Timestamp(1612556651),
address='0x5f047b29041bcfdbf0e4478cdfa753a336ba6989',
transaction_id='5bbb57386d99522d9f954c5b',
asset=Asset('LINK'),
amount=AssetAmount(FVal('1000')),
fee_asset=Asset('LINK'),
fee=Fee(FVal('0.01')),
link='',
),
AssetMovement(
location=Location.KUCOIN,
category=AssetMovementCategory.WITHDRAWAL,
timestamp=Timestamp(1612556652),
address='1DrT5xUaJ3CBZPDeFR2qdjppM6dzs4rsMt',
transaction_id=
'b893c3ece1b8d7cacb49a39ddd759cf407817f6902f566c443ba16614874ada4',
asset=Asset('BSV'),
amount=AssetAmount(FVal('2.5')),
fee_asset=Asset('BSV'),
fee=Fee(FVal('0.25')),
link='5c2dc64e03aa675aa263f1a4',
),
]
def get_endpoints_response():
results = [
f'{deposits_response_1}',
f'{deposits_response_2}',
f'{withdrawals_response_1}',
f'{withdrawals_response_2}',
# if pagination works as expected and the requesting loop is broken,
# the response below won't be processed
f'{withdrawals_response_3}',
]
for result_ in results:
yield result_
def mock_api_query_response(case, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
def _import_cryptocom_double_entries(self, data: Any, double_type: str) -> None:
"""Look for events that have double entries and handle them as trades.
This method looks for `*_debited` and `*_credited` entries using the
same timestamp to handle them as one trade.
Known double_type: 'dynamic_coin_swap' or 'dust_conversion'
"""
double_rows: Dict[Any, Dict[str, Any]] = {}
debited_row = None
credited_row = None
for row in data:
if row['Transaction Kind'] == f'{double_type}_debited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['debited'] = row
elif row['Transaction Kind'] == f'{double_type}_credited':
timestamp = deserialize_timestamp_from_date(
date=row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
if timestamp not in double_rows:
double_rows[timestamp] = {}
double_rows[timestamp]['credited'] = row
for timestamp in double_rows:
credited_row = double_rows[timestamp]['credited']
debited_row = double_rows[timestamp]['debited']
if credited_row is not None and debited_row is not None:
description = credited_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees here
fee = Fee(ZERO)
fee_currency = A_USD
base_asset = Asset(credited_row['Currency'])
quote_asset = Asset(debited_row['Currency'])
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(credited_row['Amount'])
quote_amount_sold = deserialize_asset_amount(debited_row['Amount'])
rate = Price(abs(base_amount_bought / quote_amount_sold))
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=TradeType.BUY,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
def asset_from_cryptocompare(cc_name: str) -> Asset:
return Asset(CRYPTOCOMPARE_TO_WORLD[cc_name])
def asset_from_poloniex(poloniex_name: str) -> Asset:
if poloniex_name in UNSUPPORTED_POLONIEX_ASSETS:
raise UnsupportedAsset(poloniex_name)
our_name = POLONIEX_TO_WORLD.get(poloniex_name, poloniex_name)
return Asset(our_name)
def create_fake_data(self, args: argparse.Namespace) -> None:
self._clean_tables()
from_ts, to_ts = StatisticsFaker._get_timestamps(args)
starting_amount, min_amount, max_amount = StatisticsFaker._get_amounts(
args)
total_amount = starting_amount
locations = [
deserialize_location(location)
for location in args.locations.split(',')
]
assets = [Asset(symbol) for symbol in args.assets.split(',')]
go_up_probability = FVal(args.go_up_probability)
# Add the first distribution of location data
location_data = []
for idx, value in enumerate(
divide_number_in_parts(starting_amount, len(locations))):
location_data.append(
LocationData(
time=from_ts,
location=locations[idx].serialize_for_db(),
usd_value=str(value),
))
# add the location data + total to the DB
self.db.add_multiple_location_data(location_data + [
LocationData(
time=from_ts,
location=Location.TOTAL.serialize_for_db(),
usd_value=str(total_amount),
)
])
# Add the first distribution of assets
assets_data = []
for idx, value in enumerate(
divide_number_in_parts(starting_amount, len(assets))):
assets_data.append(
AssetBalance(
time=from_ts,
asset=assets[idx],
amount=str(random.randint(1, 20)),
usd_value=str(value),
))
self.db.add_multiple_balances(assets_data)
while from_ts < to_ts:
print(
f'At timestamp: {from_ts}/{to_ts} wih total net worth: ${total_amount}'
)
new_location_data = []
new_assets_data = []
from_ts += args.seconds_between_balance_save
# remaining_loops = to_ts - from_ts / args.seconds_between_balance_save
add_usd_value = random.choice([100, 350, 500, 625, 725, 915, 1000])
add_amount = random.choice([
FVal('0.1'),
FVal('0.23'),
FVal('0.34'),
FVal('0.69'),
FVal('1.85'),
FVal('2.54'),
])
go_up = (
# If any asset's usd value is close to to go below zero, go up
any(
FVal(a.usd_value) - FVal(add_usd_value) < 0
for a in assets_data) or
# If total is going under the min amount go up
total_amount - add_usd_value < min_amount or
# If "dice roll" matched and we won't go over the max amount go up
(add_usd_value + total_amount < max_amount
and FVal(random.random()) <= go_up_probability))
if go_up:
total_amount += add_usd_value
action = operator.add
else:
total_amount -= add_usd_value
action = operator.sub
for idx, value in enumerate(
divide_number_in_parts(add_usd_value, len(locations))):
new_location_data.append(
LocationData(
time=from_ts,
location=location_data[idx].location,
usd_value=str(
action(FVal(location_data[idx].usd_value), value)),
))
# add the location data + total to the DB
self.db.add_multiple_location_data(new_location_data + [
LocationData(
time=from_ts,
location=Location.TOTAL.serialize_for_db(),
usd_value=str(total_amount),
)
])
for idx, value in enumerate(
divide_number_in_parts(add_usd_value, len(assets))):
old_amount = FVal(assets_data[idx].amount)
new_amount = action(old_amount, add_amount)
if new_amount < FVal('0'):
new_amount = old_amount + FVal('0.01')
new_assets_data.append(
AssetBalance(
time=from_ts,
asset=assets[idx],
amount=str(new_amount),
usd_value=str(
action(FVal(assets_data[idx].usd_value), value)),
))
self.db.add_multiple_balances(new_assets_data)
location_data = new_location_data
assets_data = new_assets_data
def asset_from_kraken(kraken_name: str) -> Asset:
name = KRAKEN_TO_WORLD.get(kraken_name, kraken_name)
return Asset(name)
def _atoken_to_reserve_asset(atoken: EthereumToken) -> Asset:
reserve_symbol = atoken.identifier[1:]
if reserve_symbol == 'SUSD':
reserve_symbol = 'sUSD'
return Asset(reserve_symbol)
from rotkehlchen.assets.asset import Asset, EthereumToken
A_USD = Asset('USD')
A_EUR = Asset('EUR')
A_GBP = Asset('GBP')
A_JPY = Asset('JPY')
A_CNY = Asset('CNY')
A_CAD = Asset('CAD')
A_KRW = Asset('KRW')
A_RUB = Asset('RUB')
A_CHF = Asset('CHF')
A_TRY = Asset('TRY')
A_ZAR = Asset('ZAR')
A_AUD = Asset('AUD')
A_NZD = Asset('NZD')
A_BRL = Asset('BRL')
FIAT_CURRENCIES = (
A_USD,
A_EUR,
A_GBP,
A_JPY,
A_CNY,
A_CAD,
A_KRW,
A_RUB,
A_CHF,
A_TRY,
A_ZAR,
A_AUD,
A_NZD,
A_BRL,
def _consume_cointracking_entry(self, csv_row: List[str]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row[1] # Type
timestamp = deserialize_timestamp_from_date(
date=csv_row[9],
formatstr='%d.%m.%Y %H:%M',
location='cointracking.info',
)
notes = csv_row[8]
location = exchange_row_to_location(csv_row[6])
if row_type == 'Gift/Tip' or row_type == 'Trade':
base_asset = Asset(csv_row[3])
quote_asset = None if csv_row[5] == '' else Asset(csv_row[5])
if not quote_asset and row_type != 'Gift/Tip':
raise DeserializationError(
'Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(
f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row[2])
if csv_row[4] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row[4])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.
BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=Fee(
ZERO), # There are no fees when import from cointracking
fee_currency=base_asset,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type == 'Deposit' or row_type == 'Withdrawal':
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row[2])
asset = Asset(csv_row[3])
else:
amount = deserialize_asset_amount(csv_row[4])
asset = Asset(csv_row[5])
asset_movement = AssetMovement(
location=location,
category=category,
timestamp=timestamp,
asset=asset,
amount=amount,
fee_asset=asset,
fee=Fee(ZERO),
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry', )
def test_deserialize_accounts_balances(mock_kucoin, inquirer): # pylint: disable=unused-argument
accounts_data = [
{
'id': '601ac6f7d48f8000063ab2da',
'currency': 'UNEXISTINGSYMBOL',
'type': 'main',
'balance': '999',
'available': '999',
'holds': '0',
},
{
'id': '601ac6f7d48f8000063ab2db',
'currency': 'BCHSV',
'type': 'main',
'balance': '1',
'available': '1',
'holds': '0',
},
{
'id': '601ac6f7d48f8000063ab2de',
'currency': 'BTC',
'type': 'main',
'balance': '2.52',
'available': '2.52',
'holds': '0',
},
{
'id': '601ac6f7d48f8000063ab2e7',
'currency': 'ETH',
'type': 'main',
'balance': '47.33',
'available': '47.33',
'holds': '0',
},
{
'id': '601ac6f7d48f8000063ab2e1',
'currency': 'USDT',
'type': 'main',
'balance': '34500',
'available': '34500',
'holds': '0',
},
{
'id': '60228f81d48f8000060cec67',
'currency': 'USDT',
'type': 'margin',
'balance': '10000',
'available': '10000',
'holds': '0',
},
{
'id': '601acdb7d48f8000063c6d4a',
'currency': 'BTC',
'type': 'trade',
'balance': '0.09018067',
'available': '0.09018067',
'holds': '0',
},
{
'id': '601acdc5d48f8000063c70b3',
'currency': 'USDT',
'type': 'trade',
'balance': '597.26244755',
'available': '597.26244755',
'holds': '0',
},
{
'id': '601da9fad48f8000063960cc',
'currency': 'KCS',
'type': 'trade',
'balance': '0.2',
'available': '0.2',
'holds': '0',
},
{
'id': '601da9ddd48f80000639553f',
'currency': 'ETH',
'type': 'trade',
'balance': '0.10934995',
'available': '0.10934995',
'holds': '0',
},
]
assets_balance = mock_kucoin._deserialize_accounts_balances(
{'data': accounts_data})
assert assets_balance == {
Asset('BTC'):
Balance(
amount=FVal('2.61018067'),
usd_value=FVal('3.915271005'),
),
Asset('ETH'):
Balance(
amount=FVal('47.43934995'),
usd_value=FVal('71.159024925'),
),
Asset('KCS'):
Balance(
amount=FVal('0.2'),
usd_value=FVal('0.30'),
),
Asset('USDT'):
Balance(
amount=FVal('45097.26244755'),
usd_value=FVal('67645.893671325'),
),
Asset('BSV'):
Balance(
amount=FVal('1'),
usd_value=FVal('1.5'),
),
}
def asset_from_bittrex(bittrex_name: str) -> Asset:
if bittrex_name in UNSUPPORTED_BITTREX_ASSETS:
raise UnsupportedAsset(bittrex_name)
name = BITTREX_TO_WORLD.get(bittrex_name, bittrex_name)
return Asset(name)
def test_query_trades_sandbox(sandbox_kuckoin, inquirer): # pylint: disable=unused-argument
"""The sandbox account has 6 trades. Below a list of the trades and their
timestamps in ascending mode.
- trade 1: 1612556651 -> skipped
- trade 2: 1612556693
- trade 3: 1612556765
- trade 4: 1612556765
- trade 5: 1612556765
- trade 6: 1612556794 -> skipped
By requesting trades from 1612556693 to 1612556765, the first and last trade
should be skipped.
"""
expected_trades = [
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02934995')),
rate=Price(FVal('0.046058')),
fee=Fee(FVal('9.4625999797E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec6',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.02')),
rate=Price(FVal('0.04561')),
fee=Fee(FVal('6.3854E-7')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec5',
notes='',
),
Trade(
timestamp=Timestamp(1612556765),
location=Location.KUCOIN,
pair=TradePair('ETH_BTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.06')),
rate=Price(FVal('0.0456')),
fee=Fee(FVal('0.0000019152')),
fee_currency=Asset('BTC'),
link='601da9ddf73c300006194ec4',
notes='',
),
Trade(
timestamp=Timestamp(1612556693),
location=Location.KUCOIN,
pair=TradePair('BTC_USDT'),
trade_type=TradeType.SELL,
amount=AssetAmount(FVal('0.0013')),
rate=Price(FVal('37624.4')),
fee=Fee(FVal('0.034238204')),
fee_currency=Asset('USDT'),
link='601da995e0ee8b00063a075c',
notes='',
),
]
trades = sandbox_kuckoin.query_online_trade_history(
start_ts=Timestamp(1612556693),
end_ts=Timestamp(1612556765),
)
assert trades == expected_trades
def query_historical_fiat_exchange_rates(
from_fiat_currency: Asset,
to_fiat_currency: Asset,
timestamp: Timestamp,
) -> Optional[Price]:
assert from_fiat_currency.is_fiat(
), 'fiat currency should have been provided'
assert to_fiat_currency.is_fiat(
), 'fiat currency should have been provided'
date = timestamp_to_date(timestamp, formatstr='%Y-%m-%d')
instance = Inquirer()
rate = instance._get_cached_forex_data(date, from_fiat_currency,
to_fiat_currency)
if rate:
return rate
log.debug(
'Querying exchangeratesapi',
from_fiat_currency=from_fiat_currency.identifier,
to_fiat_currency=to_fiat_currency.identifier,
timestamp=timestamp,
)
query_str = (f'https://api.exchangeratesapi.io/{date}?'
f'base={from_fiat_currency.identifier}')
resp = retry_calls(
times=5,
location='query_exchangeratesapi',
handle_429=False,
backoff_in_seconds=0,
method_name='requests.get',
function=requests.get,
# function's arguments
url=query_str,
)
if resp.status_code != 200:
return None
try:
result = rlk_jsonloads_dict(resp.text)
except JSONDecodeError:
return None
if 'rates' not in result or to_fiat_currency.identifier not in result[
'rates']:
return None
if date not in instance._cached_forex_data:
instance._cached_forex_data[date] = {}
if from_fiat_currency not in instance._cached_forex_data[date]:
instance._cached_forex_data[date][from_fiat_currency] = {}
for key, value in result['rates'].items():
instance._cached_forex_data[date][from_fiat_currency][key] = FVal(
value)
rate = Price(FVal(result['rates'][to_fiat_currency.identifier]))
log.debug('Exchangeratesapi query succesful', rate=rate)
return rate
def _consume_cryptocom_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cryptocom entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCryptocomEntry if importing of this entry is not supported.
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Transaction Kind']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Timestamp (UTC)'],
formatstr='%Y-%m-%d %H:%M:%S',
location='crypto.com',
)
description = csv_row['Transaction Description']
notes = f'{description}\nSource: crypto.com (CSV import)'
# No fees info until (Nov 2020) on crypto.com
# fees are not displayed in the export data
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if row_type in (
'crypto_purchase',
'crypto_exchange',
'referral_gift',
'referral_bonus',
'crypto_earn_interest_paid',
'referral_card_cashback',
'card_cashback_reverted',
'reimbursement',
):
# variable mapping to raw data
currency = csv_row['Currency']
to_currency = csv_row['To Currency']
native_currency = csv_row['Native Currency']
amount = csv_row['Amount']
to_amount = csv_row['To Amount']
native_amount = csv_row['Native Amount']
trade_type = TradeType.BUY if to_currency != native_currency else TradeType.SELL
if row_type == 'crypto_exchange':
# trades crypto to crypto
base_asset = Asset(to_currency)
quote_asset = Asset(currency)
if quote_asset is None:
raise DeserializationError('Got a trade entry with an empty quote asset')
base_amount_bought = deserialize_asset_amount(to_amount)
quote_amount_sold = deserialize_asset_amount(amount)
else:
base_asset = Asset(currency)
quote_asset = Asset(native_currency)
base_amount_bought = deserialize_asset_amount(amount)
quote_amount_sold = deserialize_asset_amount(native_amount)
rate = Price(abs(quote_amount_sold / base_amount_bought))
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
trade = Trade(
timestamp=timestamp,
location=Location.CRYPTOCOM,
pair=pair,
trade_type=trade_type,
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('crypto_withdrawal', 'crypto_deposit'):
if row_type == 'crypto_withdrawal':
category = AssetMovementCategory.WITHDRAWAL
amount = deserialize_asset_amount_force_positive(csv_row['Amount'])
else:
category = AssetMovementCategory.DEPOSIT
amount = deserialize_asset_amount(csv_row['Amount'])
asset = Asset(csv_row['Currency'])
asset_movement = AssetMovement(
location=Location.CRYPTOCOM,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=asset,
link='',
)
self.db.add_asset_movements([asset_movement])
elif row_type in (
'crypto_earn_program_created',
'crypto_earn_program_withdrawn',
'lockup_lock',
'lockup_unlock',
'dynamic_coin_swap_bonus_exchange_deposit',
'crypto_wallet_swap_debited',
'crypto_wallet_swap_credited',
'lockup_swap_debited',
'lockup_swap_credited',
'lockup_swap_rebate',
'dynamic_coin_swap_bonus_exchange_deposit',
# we don't handle cryto.com exchange yet
'crypto_to_exchange_transfer',
'exchange_to_crypto_transfer',
# supercharger actions
'supercharger_deposit',
'supercharger_withdrawal',
# already handled using _import_cryptocom_double_entries
'dynamic_coin_swap_debited',
'dynamic_coin_swap_credited',
'dust_conversion_debited',
'dust_conversion_credited',
):
# those types are ignored because it doesn't affect the wallet balance
# or are not handled here
return
else:
raise UnsupportedCryptocomEntry(
f'Unknown entrype type "{row_type}" encountered during '
f'cryptocom data import. Ignoring entry',
)
def query_online_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries gemini for trades
"""
log.debug('Query gemini trade history',
start_ts=start_ts,
end_ts=end_ts)
trades = []
gemini_trades = []
for symbol in self.symbols:
gemini_trades = self._get_trades_for_symbol(
symbol=symbol,
start_ts=start_ts,
end_ts=end_ts,
)
for entry in gemini_trades:
try:
timestamp = deserialize_timestamp(entry['timestamp'])
if timestamp > end_ts:
break
trades.append(
Trade(
timestamp=timestamp,
location=Location.GEMINI,
pair=gemini_symbol_to_pair(symbol),
trade_type=deserialize_trade_type(entry['type']),
amount=deserialize_asset_amount(entry['amount']),
rate=deserialize_price(entry['price']),
fee=deserialize_fee(entry['fee_amount']),
fee_currency=Asset(entry['fee_currency']),
link=str(entry['tid']),
notes='',
))
except UnprocessableTradePair as e:
self.msg_aggregator.add_warning(
f'Found unprocessable Gemini pair {e.pair}. Ignoring the trade.',
)
continue
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'Found unknown Gemini asset {e.asset_name}. '
f'Ignoring the trade.', )
continue
except (DeserializationError, KeyError) as e:
msg = str(e)
if isinstance(e, KeyError):
msg = f'Missing key entry for {msg}.'
self.msg_aggregator.add_error(
'Failed to deserialize a gemini trade. '
'Check logs for details. Ignoring it.', )
log.error(
'Error processing a gemini trade.',
raw_trade=entry,
error=msg,
)
continue
return trades
def _consume_cointracking_entry(self, csv_row: Dict[str, Any]) -> None:
"""Consumes a cointracking entry row from the CSV and adds it into the database
Can raise:
- DeserializationError if something is wrong with the format of the expected values
- UnsupportedCointrackingEntry if importing of this entry is not supported.
- IndexError if the CSV file is corrupt
- KeyError if the an expected CSV key is missing
- UnknownAsset if one of the assets founds in the entry are not supported
"""
row_type = csv_row['Type']
timestamp = deserialize_timestamp_from_date(
date=csv_row['Date'],
formatstr='%d.%m.%Y %H:%M:%S',
location='cointracking.info',
)
notes = csv_row['Comment']
location = exchange_row_to_location(csv_row['Exchange'])
fee = Fee(ZERO)
fee_currency = A_USD # whatever (used only if there is no fee)
if csv_row['Fee'] != '':
fee = deserialize_fee(csv_row['Fee'])
fee_currency = Asset(csv_row['Cur.Fee'])
if row_type in ('Gift/Tip', 'Trade', 'Income'):
base_asset = Asset(csv_row['Cur.Buy'])
quote_asset = None if csv_row['Cur.Sell'] == '' else Asset(csv_row['Cur.Sell'])
if quote_asset is None and row_type not in ('Gift/Tip', 'Income'):
raise DeserializationError('Got a trade entry with an empty quote asset')
if quote_asset is None:
# Really makes no difference as this is just a gift and the amount is zero
quote_asset = A_USD
pair = TradePair(f'{base_asset.identifier}_{quote_asset.identifier}')
base_amount_bought = deserialize_asset_amount(csv_row['Buy'])
if csv_row['Sell'] != '-':
quote_amount_sold = deserialize_asset_amount(csv_row['Sell'])
else:
quote_amount_sold = AssetAmount(ZERO)
rate = Price(quote_amount_sold / base_amount_bought)
trade = Trade(
timestamp=timestamp,
location=location,
pair=pair,
trade_type=TradeType.BUY, # It's always a buy during cointracking import
amount=base_amount_bought,
rate=rate,
fee=fee,
fee_currency=fee_currency,
link='',
notes=notes,
)
self.db.add_trades([trade])
elif row_type in ('Deposit', 'Withdrawal'):
category = deserialize_asset_movement_category(row_type.lower())
if category == AssetMovementCategory.DEPOSIT:
amount = deserialize_asset_amount(csv_row['Buy'])
asset = Asset(csv_row['Cur.Buy'])
else:
amount = deserialize_asset_amount_force_positive(csv_row['Sell'])
asset = Asset(csv_row['Cur.Sell'])
asset_movement = AssetMovement(
location=location,
category=category,
address=None,
transaction_id=None,
timestamp=timestamp,
asset=asset,
amount=amount,
fee=fee,
fee_asset=fee_currency,
link='',
)
self.db.add_asset_movements([asset_movement])
else:
raise UnsupportedCointrackingEntry(
f'Unknown entrype type "{row_type}" encountered during cointracking '
f'data import. Ignoring entry',
)
def test_deserialize_trade_buy(mock_bitstamp):
raw_trade = {
'id': 2,
'type': 2,
'datetime': '2020-12-02 09:30:00',
'btc': '0.50000000',
'usd': '-10000.00000000',
'btc_usd': '0.00005000',
'fee': '20.00000000',
'order_id': 2,
}
expected_trade = Trade(
timestamp=1606901400,
location=Location.BITSTAMP,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=FVal('0.50000000'),
rate=FVal('0.00005000'),
fee=FVal('20.00000000'),
fee_currency=Asset('USD'),
link='2',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
raw_trade = {
'id': 2,
'type': 2,
'datetime': '2019-04-16 08:09:05.149343',
'btc': '0.00060000',
'usd': '0',
'btc_eur': '8364.0',
'eur': '-5.02',
'fee': '0.02',
'order_id': 2,
}
expected_trade = Trade(
timestamp=1555402145,
location=Location.BITSTAMP,
pair=TradePair('BTC_EUR'),
trade_type=TradeType.BUY,
amount=FVal('0.0006'),
rate=FVal('8364.0'),
fee=FVal('0.02'),
fee_currency=Asset('EUR'),
link='2',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
raw_trade = {
'id': 15,
'type': 2,
'datetime': '2019-04-15 16:19:14.826000',
'btc': '0',
'usd': '-7.70998',
'eur_usd': '1.12124',
'eur': '6.87630',
'fee': '0.02',
'order_id': 15,
}
expected_trade = Trade(
timestamp=1555345154,
location=Location.BITSTAMP,
pair=TradePair('EUR_USD'),
trade_type=TradeType.BUY,
amount=FVal('6.8763'),
rate=FVal('1.12124'),
fee=FVal('0.02'),
fee_currency=Asset('USD'),
link='15',
notes='',
)
trade = mock_bitstamp._deserialize_trade(raw_trade)
assert trade == expected_trade
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
if from_asset in KNOWN_TO_MISS_FROM_CRYPTOCOMPARE:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is None or data[index].low is None:
# If we get some None in the hourly set price to 0 so that we check alternatives
price = Price(ZERO)
else:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def test_api_query_paginated_trades_pagination(mock_bitstamp):
"""Test pagination logic for trades works as expected.
First request: 2 results, 1 valid trade (id 2)
Second request: 2 results, no trades
Third request: 2 results, 1 valid trade (id 5) and 1 invalid trade (id 6)
Trades with id 2 and 5 are expected to be returned.
"""
# Not a trade
user_transaction_1 = """
{
"id": 1,
"type": -1,
"datetime": "2020-12-02 09:00:00"
}
"""
# First trade, buy BTC with USD, within timestamp range
user_transaction_2 = """
{
"id": 2,
"type": 2,
"datetime": "2020-12-02 09:30:00",
"btc": "0.50000000",
"usd": "-10000.00000000",
"btc_usd": "0.00005000",
"fee": "20.00000000",
"order_id": 2
}
"""
# Not a trade
user_transaction_3 = """
{
"id": 3,
"type": -1,
"datetime": "2020-12-02 18:00:00"
}
"""
# Not a trade
user_transaction_4 = """
{
"id": 4,
"type": -1,
"datetime": "2020-12-03 9:00:00"
}
"""
# Second trade, sell EUR for USD, within timestamp range
user_transaction_5 = """
{
"id": 5,
"type": 2,
"datetime": "2020-12-03 11:30:00",
"eur": "-1.00000000",
"usd": "1.22000000",
"eur_usd": "0.81967213",
"fee": "0.00610000",
"order_id": 3
}
"""
# Third trade, buy ETH with USDC, out of timestamp range
user_transaction_6 = """
{
"id": 6,
"type": 2,
"datetime": "2020-12-03 12:00:01",
"eth": "1.00000000",
"usdc": "-750.00000000",
"eth_usdc": "0.00133333",
"fee": "3.75000000",
"order_id": 1
}
"""
api_limit = 2
now = datetime.now()
now_ts = int(now.timestamp())
options = {
'since_id': USER_TRANSACTION_MIN_SINCE_ID,
'limit': api_limit,
'sort': USER_TRANSACTION_SORTING_MODE,
'offset': 0,
}
expected_calls = [
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 1,
'limit': 2,
'sort': 'asc',
'offset': 0,
},
),
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 3,
'limit': 2,
'sort': 'asc',
'offset': 0,
},
),
call(
endpoint='user_transactions',
method='post',
options={
'since_id': 3,
'limit': 2,
'sort': 'asc',
'offset': 2,
},
),
]
def get_paginated_response():
results = [
f'[{user_transaction_1},{user_transaction_2}]',
f'[{user_transaction_3},{user_transaction_4}]',
f'[{user_transaction_5},{user_transaction_6}]',
]
for result_ in results:
yield result_
def mock_api_query_response(endpoint, method, options): # pylint: disable=unused-argument
return MockResponse(HTTPStatus.OK, next(get_response))
get_response = get_paginated_response()
with patch(
'rotkehlchen.exchanges.bitstamp.API_MAX_LIMIT',
new_callable=MagicMock(return_value=api_limit),
):
with patch.object(
mock_bitstamp,
'_api_query',
side_effect=mock_api_query_response,
) as mock_api_query:
result = mock_bitstamp._api_query_paginated(
start_ts=Timestamp(0),
end_ts=Timestamp(now_ts),
options=options,
case='trades',
)
assert mock_api_query.call_args_list == expected_calls
expected_result = [
Trade(
timestamp=1606901400,
location=Location.BITSTAMP,
pair=TradePair('BTC_USD'),
trade_type=TradeType.BUY,
amount=FVal('0.50000000'),
rate=FVal('0.00005000'),
fee=FVal('20.00000000'),
fee_currency=Asset('USD'),
link='2',
notes='',
),
Trade(
timestamp=1606995000,
location=Location.BITSTAMP,
pair=TradePair('EUR_USD'),
trade_type=TradeType.SELL,
amount=FVal('1'),
rate=FVal('0.81967213'),
fee=FVal('0.00610000'),
fee_currency=Asset('USD'),
link='5',
notes='',
),
]
assert result == expected_result
def test_unknown_asset():
"""Test than an unknown asset will throw"""
with pytest.raises(UnknownAsset):
Asset('jsakdjsladjsakdj')
from rotkehlchen.assets.asset import Asset, EthereumToken
from rotkehlchen.serialization.deserialize import deserialize_ethereum_address
A_RDN = EthereumToken('RDN')
A_GNO = EthereumToken('GNO')
A_DAO = EthereumToken('DAO')
A_MKR = EthereumToken('MKR')
A_SNGLS = EthereumToken('SNGLS')
A_USDT = EthereumToken('USDT')
A_MKR = EthereumToken('MKR')
A_BAT = EthereumToken('BAT')
A_WBTC = EthereumToken('WBTC')
A_USDC = EthereumToken('USDC')
A_ADAI = Asset('aDAI')
A_CDAI = Asset('cDAI')
A_CUSDC = Asset('cUSDC')
A_DOGE = Asset('DOGE')
A_LTC = Asset('LTC')
A_XMR = Asset('XMR')
A_DASH = Asset('DASH')
A_IOTA = Asset('IOTA')
A_BSV = Asset('BSV')
A_BCH = Asset('BCH')
A_BNB = Asset('BNB')
A_CNY = Asset('CNY')
A_JPY = Asset('JPY')
A_ZEC = Asset('ZEC')
A_BUSD = Asset('BUSD')
A_DOT = Asset('DOT')
A_GBP = Asset('GBP')
def get_ignored_assets(self) -> List[Asset]:
cursor = self.conn.cursor()
cursor.execute(
'SELECT value FROM multisettings WHERE name="ignored_asset";', )
return [Asset(q[0]) for q in cursor]
def aave_event_from_db(event_tuple: AAVE_EVENT_DB_TUPLE) -> AaveEvent:
"""Turns a tuple read from the DB into an appropriate AaveEvent
May raise a DeserializationError if something is wrong with the DB data
"""
event_type = event_tuple[1]
block_number = event_tuple[2]
timestamp = Timestamp(event_tuple[3])
tx_hash = event_tuple[4]
log_index = event_tuple[5]
try:
asset1 = Asset(event_tuple[6])
except UnknownAsset as e:
raise DeserializationError(
f'Unknown asset {event_tuple[6]} encountered during deserialization '
f'of Aave event from DB',
) from e
asset1_amount = FVal(event_tuple[7])
asset1_usd_value = FVal(event_tuple[8])
if event_type in ('deposit', 'withdrawal', 'interest'):
return AaveSimpleEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
)
if event_type == 'borrow':
if event_tuple[12] not in ('stable', 'variable'):
raise DeserializationError(
f'Invalid borrow rate mode encountered in the DB: {event_tuple[12]}',
)
borrow_rate_mode: Literal['stable', 'variable'] = event_tuple[12] # type: ignore
borrow_rate = deserialize_optional_fval(
value=event_tuple[10],
name='borrow_rate',
location='reading aave borrow event from DB',
)
accrued_borrow_interest = deserialize_optional_fval(
value=event_tuple[11],
name='accrued_borrow_interest',
location='reading aave borrow event from DB',
)
return AaveBorrowEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
borrow_rate_mode=borrow_rate_mode,
borrow_rate=borrow_rate,
accrued_borrow_interest=accrued_borrow_interest,
)
if event_type == 'repay':
fee_amount = deserialize_optional_fval(
value=event_tuple[10],
name='fee_amount',
location='reading aave repay event from DB',
)
fee_usd_value = deserialize_optional_fval(
value=event_tuple[11],
name='fee_usd_value',
location='reading aave repay event from DB',
)
return AaveRepayEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
asset=asset1,
value=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
fee=Balance(amount=fee_amount, usd_value=fee_usd_value),
)
if event_type == 'liquidation':
try:
# If event_tuple[9] is None DeserializationError is raised from Asset ctor->type ignore
principal_asset = Asset(event_tuple[9]) # type: ignore
except UnknownAsset as e:
raise DeserializationError(
f'Unknown asset {event_tuple[6]} encountered during deserialization '
f'of Aave event from DB',
) from e
principal_amount = deserialize_optional_fval(
value=event_tuple[10],
name='principal_amount',
location='reading aave liquidation event from DB',
)
principal_usd_value = deserialize_optional_fval(
value=event_tuple[11],
name='principal_usd_value',
location='reading aave liquidation event from DB',
)
return AaveLiquidationEvent(
event_type=event_type,
block_number=block_number,
timestamp=timestamp,
tx_hash=tx_hash,
log_index=log_index,
collateral_asset=asset1,
collateral_balance=Balance(amount=asset1_amount, usd_value=asset1_usd_value),
principal_asset=principal_asset,
principal_balance=Balance(
amount=principal_amount,
usd_value=principal_usd_value,
),
)
# else
raise DeserializationError(
f'Unknown event type {event_type} encountered during '
f'deserialization of Aave event from DB',
)
price = cryptocompare.query_historical_price(
from_asset=Asset('DAO'),
to_asset=A_USD,
timestamp=1468886400,
historical_data_start=1438387200,
)
assert price is not None
@pytest.mark.skipif(
'CI' in os.environ,
reason='This test would heavily contribute in cryptocompare rate limiting',
)
@pytest.mark.parametrize('run', (
[{
'asset': Asset('cDAI'),
'expected_price1': FVal('0.02012010'),
'expected_price2': FVal('0.02033108'),
}, {
'asset': Asset('cBAT'),
'expected_price1': FVal('0.003522603'),
'expected_price2': FVal('0.002713524'),
}, {
'asset': Asset('cETH'),
'expected_price1': FVal('2.903'),
'expected_price2': FVal('2.669'),
}, {
'asset': Asset('cREP'),
'expected_price1': FVal('0.20105130'),
'expected_price2': FVal('0.16380696'),
}, {
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
May raise:
- PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
try:
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
except UnsupportedAsset as e:
raise PriceQueryUnsupportedAsset(e.asset_name)
price = Price(ZERO)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index_in_bounds = True
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
if index > len(data) - 1: # index out of bounds
# Try to see if index - 1 is there and if yes take it
if index > len(data):
index = index - 1
else: # give up. This happened: https://github.com/rotki/rotki/issues/1534
log.error(
f'Expected data index in cryptocompare historical hour price '
f'not found. Queried price of: {from_asset.identifier} in '
f'{to_asset.identifier} at {timestamp}. Data '
f'index: {index}. Length of returned data: {len(data)}. '
f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...',
)
index_in_bounds = False
if index_in_bounds:
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is not None and data[index].low is not None:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=Asset('BTC'),
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=Asset('BTC'),
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = (
(to_asset.is_fiat() and to_asset != Asset('USD'))
or (from_asset.is_fiat() and from_asset != Asset('USD')))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
