def test_deserialize_location(database):
balances = []
for idx, data in enumerate(Location):
assert deserialize_location(str(data)) == data
balances.append(
ManuallyTrackedBalance(
asset=A_BTC,
label='Test' + str(idx),
amount=FVal(1),
location=data,
tags=None,
))
with pytest.raises(DeserializationError):
deserialize_location('dsadsad')
with pytest.raises(DeserializationError):
deserialize_location(15)
# Also write and read each location to DB to make sure that
# location.serialize_for_db() and deserialize_location_from_db work fine
add_manually_tracked_balances(database, balances)
balances = database.get_manually_tracked_balances()
for data in Location:
assert data in (x.location for x in balances)
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[AssetMovement]:
"""Queries the local DB and the exchange for the deposits/withdrawal history of the user"""
asset_movements = self.db.get_asset_movements(
from_ts=start_ts,
to_ts=end_ts,
location=deserialize_location(self.name),
)
ranges = DBQueryRanges(self.db)
ranges_to_query = ranges.get_location_query_ranges(
location_string=f'{self.name}_asset_movements',
start_ts=start_ts,
end_ts=end_ts,
)
new_movements = []
for query_start_ts, query_end_ts in ranges_to_query:
new_movements.extend(self.query_online_deposits_withdrawals(
start_ts=query_start_ts,
end_ts=query_end_ts,
))
if new_movements != []:
self.db.add_asset_movements(new_movements)
ranges.update_used_query_range(
location_string=f'{self.name}_asset_movements',
start_ts=start_ts,
end_ts=end_ts,
ranges_to_query=ranges_to_query,
)
asset_movements.extend(new_movements)
return asset_movements
def _query_exchange_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
all_movements: List[AssetMovement],
exchange: ExchangeInterface,
) -> List[AssetMovement]:
location = deserialize_location(exchange.name)
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
location_movements = exchange.query_deposits_withdrawals(
start_ts=from_ts, end_ts=to_ts)
movements: List[AssetMovement] = []
if self.premium is None:
movements = self._apply_actions_limit(
location=location,
action_type='asset_movement',
location_actions=location_movements,
all_actions=all_movements,
)
else:
movements = location_movements
return movements
def deserialize_trade(data: Dict[str, Any]) -> Trade:
"""
Takes a dict trade representation of our common trade format and serializes
it into the Trade object
May raise:
- UnknownAsset: If the fee_currency string is not a known asset
- DeserializationError: If any of the trade dict entries is not as expected
"""
pair = data['pair']
rate = deserialize_price(data['rate'])
amount = deserialize_asset_amount(data['amount'])
trade_type = deserialize_trade_type(data['trade_type'])
location = deserialize_location(data['location'])
trade_link = ''
if 'link' in data:
trade_link = data['link']
trade_notes = ''
if 'notes' in data:
trade_notes = data['notes']
return Trade(
timestamp=data['timestamp'],
location=location,
pair=pair,
trade_type=trade_type,
amount=amount,
rate=rate,
fee=deserialize_fee(data['fee']),
fee_currency=Asset(data['fee_currency']),
link=trade_link,
notes=trade_notes,
)
def mock_exchange_data_in_db(exchanges, rotki) -> None:
db = rotki.data.db
for exchange_name in exchanges:
db.add_trades([
Trade(
timestamp=Timestamp(1),
location=deserialize_location(exchange_name),
base_asset=A_BTC,
quote_asset=A_ETH,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal(1)),
rate=Price(FVal(1)),
fee=Fee(FVal('0.1')),
fee_currency=A_ETH,
link='foo',
notes='boo',
)
])
db.update_used_query_range(name=f'{exchange_name}_trades',
start_ts=0,
end_ts=9999)
db.update_used_query_range(name=f'{exchange_name}_margins',
start_ts=0,
end_ts=9999)
db.update_used_query_range(name=f'{exchange_name}_asset_movements',
start_ts=0,
end_ts=9999) # noqa: E501
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> TRADES_LIST:
"""Queries trades for the given location and time range.
If no location is given then all external, all exchange and DEX trades are queried.
DEX Trades are queried only if the user has premium
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
trades: TRADES_LIST
if location is not None:
trades = self.query_location_trades(from_ts, to_ts, location)
else:
trades = self.query_location_trades(from_ts, to_ts,
Location.EXTERNAL)
# crypto.com is not an API key supported exchange but user can import from CSV
trades.extend(
self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM))
for name, exchange in self.exchange_manager.connected_exchanges.items(
):
exchange_trades = exchange.query_trade_history(
start_ts=from_ts, end_ts=to_ts)
if self.premium is None:
trades = self._apply_actions_limit(
location=deserialize_location(name),
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# for all trades we also need uniswap trades
if self.premium is not None:
uniswap = self.chain_manager.uniswap
if uniswap is not None:
trades.extend(
uniswap.get_trades(
addresses=self.chain_manager.
queried_addresses_for_module('uniswap'),
from_timestamp=from_ts,
to_timestamp=to_ts,
), )
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
def _deserialize(
self,
value: str,
attr: Optional[str], # pylint: disable=unused-argument
data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument
**_kwargs: Any,
) -> Location:
try:
location = deserialize_location(value)
except DeserializationError as e:
raise ValidationError(str(e))
return location
def _deserialize(
self,
value: str,
attr, # pylint: disable=unused-argument
data, # pylint: disable=unused-argument
**kwargs, # pylint: disable=unused-argument
) -> Location:
try:
location = deserialize_location(value)
except DeserializationError as e:
raise ValidationError(str(e))
return location
def query_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
only_cache: bool,
) -> List[Trade]:
"""Queries the local DB and the remote exchange for the trade history of the user
Limits the query to the given time range and also if only_cache is True returns
only what is already saved in the DB without performing an exchange query
"""
trades = self.db.get_trades(
from_ts=start_ts,
to_ts=end_ts,
location=deserialize_location(self.name),
)
if only_cache:
return trades
ranges = DBQueryRanges(self.db)
ranges_to_query = ranges.get_location_query_ranges(
location_string=f'{self.name}_trades',
start_ts=start_ts,
end_ts=end_ts,
)
new_trades = []
for query_start_ts, query_end_ts in ranges_to_query:
# If we have a time frame we have not asked the exchange for trades then
# go ahead and do that now
new_trades.extend(
self.query_online_trade_history(
start_ts=query_start_ts,
end_ts=query_end_ts,
))
# make sure to add them to the DB
if new_trades != []:
self.db.add_trades(new_trades)
# and also set the used queried timestamp range for the exchange
ranges.update_used_query_range(
location_string=f'{self.name}_trades',
start_ts=start_ts,
end_ts=end_ts,
ranges_to_query=ranges_to_query,
)
# finally append them to the already returned DB trades
trades.extend(new_trades)
return trades
def query_trade_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[Trade]:
"""Queries the local DB and the remote exchange for the trade history of the user"""
trades = self.db.get_trades(
from_ts=start_ts,
to_ts=end_ts,
location=deserialize_location(self.name),
)
ranges = DBQueryRanges(self.db)
ranges_to_query = ranges.get_location_query_ranges(
location_string=f'{self.name}_trades',
start_ts=start_ts,
end_ts=end_ts,
)
new_trades = []
for query_start_ts, query_end_ts in ranges_to_query:
# If we have a time frame we have not asked the exchange for trades then
# go ahead and do that now
try:
new_trades.extend(
self.query_online_trade_history(
start_ts=query_start_ts,
end_ts=query_end_ts,
))
except NotImplementedError:
msg = 'query_online_trade_history should only not be implemented by bitmex'
assert self.name == 'bitmex', msg
# make sure to add them to the DB
if new_trades != []:
self.db.add_trades(new_trades)
# and also set the used queried timestamp range for the exchange
ranges.update_used_query_range(
location_string=f'{self.name}_trades',
start_ts=start_ts,
end_ts=end_ts,
ranges_to_query=ranges_to_query,
)
# finally append them to the already returned DB trades
trades.extend(new_trades)
return trades
def _query_exchange_asset_movements(
self,
from_ts: Timestamp,
to_ts: Timestamp,
all_movements: List[AssetMovement],
exchange: Union[ExchangeInterface, Location],
only_cache: bool,
) -> List[AssetMovement]:
if isinstance(exchange, ExchangeInterface):
location = deserialize_location(exchange.name)
# clear the asset movements queried for this exchange
self.actions_per_location['asset_movement'][location] = 0
location_movements = exchange.query_deposits_withdrawals(
start_ts=from_ts,
end_ts=to_ts,
only_cache=only_cache,
)
else:
assert isinstance(exchange,
Location), 'only a location should make it here'
assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here'
location = exchange
# cryptocom has no exchange integration but we may have DB entries
self.actions_per_location['asset_movement'][location] = 0
location_movements = self.data.db.get_asset_movements(
from_ts=from_ts,
to_ts=to_ts,
location=location,
)
movements: List[AssetMovement] = []
if self.premium is None:
movements = self._apply_actions_limit(
location=location,
action_type='asset_movement',
location_actions=location_movements,
all_actions=all_movements,
)
else:
all_movements.extend(location_movements)
movements = all_movements
return movements
def check_saved_events_for_exchange(
exchange_name: str,
db: DBHandler,
should_exist: bool,
) -> None:
trades = db.get_trades(location=deserialize_location(exchange_name))
trades_range = db.get_used_query_range(f'{exchange_name}_trades')
margins_range = db.get_used_query_range(f'{exchange_name}_margins')
movements_range = db.get_used_query_range(
f'{exchange_name}_asset_movements')
if should_exist:
assert trades_range is not None
assert margins_range is not None
assert movements_range is not None
assert len(trades) != 0
else:
assert trades_range is None
assert margins_range is None
assert movements_range is None
assert len(trades) == 0
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
) -> List[Trade]:
"""Queries trades for the given location and time range.
If no location is given then all external and all exchange trades are queried.
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connectingto any of the remote exchanges
"""
if location is not None:
trades = self.query_location_trades(from_ts, to_ts, location)
else:
trades = self.query_location_trades(from_ts, to_ts,
Location.EXTERNAL)
for name, exchange in self.exchange_manager.connected_exchanges.items(
):
exchange_trades = exchange.query_trade_history(
start_ts=from_ts, end_ts=to_ts)
if self.premium is None:
trades = self._apply_actions_limit(
location=deserialize_location(name),
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
def create_action(self, index: int, ts: Timestamp):
"""Create a random trade action on a random exchange depending
on the funds that are available in that exchange"""
# choose an exchange at random
exchange_name = random.choice(ALLOWED_EXCHANGES)
exchange = getattr(self, exchange_name)
# choose a random pair at that exchange
pair = exchange.choose_pair(
timestamp=ts,
price_query=self.query_historical_price,
)
print(
f'Creating trade {index + 1} / {self.trades_number} in {exchange_name}'
f' for the pair: {pair} at timestamp {ts}', )
# depending on our funds decide on what to do. Buy/sell
base, quote = pair_get_assets(pair)
if exchange.get_balance(base) is None:
action_type = TradeType.BUY
elif exchange.get_balance(quote) is None:
action_type = TradeType.SELL
else:
# TODO: trade the one we have most of
action_type = random.choice(list(TradeType))
# if we are buying we are going to spend from the quote asset
if action_type == TradeType.BUY:
spending_asset = quote
else: # selling spends from the base asset
spending_asset = base
# get a spending asset amount within our per-trade equivalent range and
# our available funds
spending_usd_rate = self.query_historical_price(
spending_asset, A_USD, ts)
max_usd_in_spending_asset = spending_usd_rate * exchange.get_balance(
spending_asset)
max_usd_equivalent_to_spend = min(max_usd_in_spending_asset,
MAX_TRADE_USD_VALUE)
rate = self.query_historical_price(base, quote, ts)
usd_to_spend = FVal(
random.uniform(0.01, float(max_usd_equivalent_to_spend)))
amount_in_spending_asset = usd_to_spend / spending_usd_rate
# if we are buying then the amount is the amount of asset we bought
if action_type == TradeType.BUY:
amount = amount_in_spending_asset / rate
# if we are selling the amount is the spending asset amount
else:
amount = amount_in_spending_asset
quote_asset_usd_rate = self.query_historical_price(quote, A_USD, ts)
fee_in_quote_currency = FVal(random.uniform(
0, MAX_FEE_USD_VALUE)) / quote_asset_usd_rate
# create the trade
trade = Trade(
timestamp=ts,
location=deserialize_location(exchange_name),
pair=pair,
trade_type=action_type,
amount=amount,
rate=rate,
fee=fee_in_quote_currency,
fee_currency=quote,
link='',
notes='',
)
logger.info(f'Created trade: {trade}')
# Adjust our global and per exchange accounting
if action_type == TradeType.BUY:
# we buy so we increase our base asset by amount
self.increase_asset(base, amount, exchange_name)
# and decrease quote by amount * rate
self.decrease_asset(quote, amount * rate, exchange_name)
else:
# we sell so we increase our quote asset
self.increase_asset(quote, amount * rate, exchange_name)
# and decrease our base asset
self.decrease_asset(base, amount, exchange_name)
# finally add it to the exchange
exchange.append_trade(trade)
def create_fake_data(self, args: argparse.Namespace) -> None:
self._clean_tables()
from_ts, to_ts = StatisticsFaker._get_timestamps(args)
starting_amount, min_amount, max_amount = StatisticsFaker._get_amounts(
args)
total_amount = starting_amount
locations = [
deserialize_location(location)
for location in args.locations.split(',')
]
assets = [Asset(symbol) for symbol in args.assets.split(',')]
go_up_probability = FVal(args.go_up_probability)
# Add the first distribution of location data
location_data = []
for idx, value in enumerate(
divide_number_in_parts(starting_amount, len(locations))):
location_data.append(
LocationData(
time=from_ts,
location=locations[idx].serialize_for_db(),
usd_value=str(value),
))
# add the location data + total to the DB
self.db.add_multiple_location_data(location_data + [
LocationData(
time=from_ts,
location=Location.TOTAL.serialize_for_db(),
usd_value=str(total_amount),
)
])
# Add the first distribution of assets
assets_data = []
for idx, value in enumerate(
divide_number_in_parts(starting_amount, len(assets))):
assets_data.append(
DBAssetBalance(
category=BalanceType.ASSET,
time=from_ts,
asset=assets[idx],
amount=str(random.randint(1, 20)),
usd_value=str(value),
))
self.db.add_multiple_balances(assets_data)
while from_ts < to_ts:
print(
f'At timestamp: {from_ts}/{to_ts} wih total net worth: ${total_amount}'
)
new_location_data = []
new_assets_data = []
from_ts += args.seconds_between_balance_save
# remaining_loops = to_ts - from_ts / args.seconds_between_balance_save
add_usd_value = random.choice([100, 350, 500, 625, 725, 915, 1000])
add_amount = random.choice([
FVal('0.1'),
FVal('0.23'),
FVal('0.34'),
FVal('0.69'),
FVal('1.85'),
FVal('2.54'),
])
go_up = (
# If any asset's usd value is close to to go below zero, go up
any(
FVal(a.usd_value) - FVal(add_usd_value) < 0
for a in assets_data) or
# If total is going under the min amount go up
total_amount - add_usd_value < min_amount or
# If "dice roll" matched and we won't go over the max amount go up
(add_usd_value + total_amount < max_amount
and FVal(random.random()) <= go_up_probability))
if go_up:
total_amount += add_usd_value
action = operator.add
else:
total_amount -= add_usd_value
action = operator.sub
for idx, value in enumerate(
divide_number_in_parts(add_usd_value, len(locations))):
new_location_data.append(
LocationData(
time=from_ts,
location=location_data[idx].location,
usd_value=str(
action(FVal(location_data[idx].usd_value), value)),
))
# add the location data + total to the DB
self.db.add_multiple_location_data(new_location_data + [
LocationData(
time=from_ts,
location=Location.TOTAL.serialize_for_db(),
usd_value=str(total_amount),
)
])
for idx, value in enumerate(
divide_number_in_parts(add_usd_value, len(assets))):
old_amount = FVal(assets_data[idx].amount)
new_amount = action(old_amount, add_amount)
if new_amount < FVal('0'):
new_amount = old_amount + FVal('0.01')
new_assets_data.append(
DBAssetBalance(
category=BalanceType.ASSET,
time=from_ts,
asset=assets[idx],
amount=str(new_amount),
usd_value=str(
action(FVal(assets_data[idx].usd_value), value)),
))
self.db.add_multiple_balances(new_assets_data)
location_data = new_location_data
assets_data = new_assets_data
def query_trades(
self,
from_ts: Timestamp,
to_ts: Timestamp,
location: Optional[Location],
only_cache: bool,
) -> TRADES_LIST:
"""Queries trades for the given location and time range.
If no location is given then all external, all exchange and DEX trades are queried.
If only_cache is given then only trades cached in the DB are returned.
No service is queried.
DEX Trades are queried only if the user has premium
If the user does not have premium then a trade limit is applied.
May raise:
- RemoteError: If there are problems connecting to any of the remote exchanges
"""
trades: TRADES_LIST
if location is not None:
trades = self.query_location_trades(from_ts, to_ts, location,
only_cache)
else:
trades = self.query_location_trades(from_ts, to_ts,
Location.EXTERNAL, only_cache)
# crypto.com is not an API key supported exchange but user can import from CSV
trades.extend(
self.query_location_trades(
from_ts=from_ts,
to_ts=to_ts,
location=Location.CRYPTOCOM,
only_cache=only_cache,
))
for name, exchange in self.exchange_manager.connected_exchanges.items(
):
exchange_trades = exchange.query_trade_history(
start_ts=from_ts,
end_ts=to_ts,
only_cache=only_cache,
)
if self.premium is None:
trades = self._apply_actions_limit(
location=deserialize_location(name),
action_type='trade',
location_actions=exchange_trades,
all_actions=trades,
)
else:
trades.extend(exchange_trades)
# for all trades we also need the trades from the amm protocols
if self.premium is not None:
for amm_location in AMMTradeLocations:
amm_module_name = cast(AMMTRADE_LOCATION_NAMES,
str(amm_location))
amm_module = self.chain_manager.get_module(amm_module_name)
if amm_module is not None:
trades.extend(
amm_module.get_trades(
addresses=self.chain_manager.
queried_addresses_for_module(
amm_module_name), # noqa: E501
from_timestamp=from_ts,
to_timestamp=to_ts,
only_cache=only_cache,
), )
# return trades with most recent first
trades.sort(key=lambda x: x.timestamp, reverse=True)
return trades
