def test_deserialize_location(database): balances = [] for idx, data in enumerate(Location): assert deserialize_location(str(data)) == data balances.append( ManuallyTrackedBalance( asset=A_BTC, label='Test' + str(idx), amount=FVal(1), location=data, tags=None, )) with pytest.raises(DeserializationError): deserialize_location('dsadsad') with pytest.raises(DeserializationError): deserialize_location(15) # Also write and read each location to DB to make sure that # location.serialize_for_db() and deserialize_location_from_db work fine add_manually_tracked_balances(database, balances) balances = database.get_manually_tracked_balances() for data in Location: assert data in (x.location for x in balances)
def query_deposits_withdrawals( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[AssetMovement]: """Queries the local DB and the exchange for the deposits/withdrawal history of the user""" asset_movements = self.db.get_asset_movements( from_ts=start_ts, to_ts=end_ts, location=deserialize_location(self.name), ) ranges = DBQueryRanges(self.db) ranges_to_query = ranges.get_location_query_ranges( location_string=f'{self.name}_asset_movements', start_ts=start_ts, end_ts=end_ts, ) new_movements = [] for query_start_ts, query_end_ts in ranges_to_query: new_movements.extend(self.query_online_deposits_withdrawals( start_ts=query_start_ts, end_ts=query_end_ts, )) if new_movements != []: self.db.add_asset_movements(new_movements) ranges.update_used_query_range( location_string=f'{self.name}_asset_movements', start_ts=start_ts, end_ts=end_ts, ranges_to_query=ranges_to_query, ) asset_movements.extend(new_movements) return asset_movements
def _query_exchange_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, all_movements: List[AssetMovement], exchange: ExchangeInterface, ) -> List[AssetMovement]: location = deserialize_location(exchange.name) # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 location_movements = exchange.query_deposits_withdrawals( start_ts=from_ts, end_ts=to_ts) movements: List[AssetMovement] = [] if self.premium is None: movements = self._apply_actions_limit( location=location, action_type='asset_movement', location_actions=location_movements, all_actions=all_movements, ) else: movements = location_movements return movements
def deserialize_trade(data: Dict[str, Any]) -> Trade: """ Takes a dict trade representation of our common trade format and serializes it into the Trade object May raise: - UnknownAsset: If the fee_currency string is not a known asset - DeserializationError: If any of the trade dict entries is not as expected """ pair = data['pair'] rate = deserialize_price(data['rate']) amount = deserialize_asset_amount(data['amount']) trade_type = deserialize_trade_type(data['trade_type']) location = deserialize_location(data['location']) trade_link = '' if 'link' in data: trade_link = data['link'] trade_notes = '' if 'notes' in data: trade_notes = data['notes'] return Trade( timestamp=data['timestamp'], location=location, pair=pair, trade_type=trade_type, amount=amount, rate=rate, fee=deserialize_fee(data['fee']), fee_currency=Asset(data['fee_currency']), link=trade_link, notes=trade_notes, )
def mock_exchange_data_in_db(exchanges, rotki) -> None: db = rotki.data.db for exchange_name in exchanges: db.add_trades([ Trade( timestamp=Timestamp(1), location=deserialize_location(exchange_name), base_asset=A_BTC, quote_asset=A_ETH, trade_type=TradeType.BUY, amount=AssetAmount(FVal(1)), rate=Price(FVal(1)), fee=Fee(FVal('0.1')), fee_currency=A_ETH, link='foo', notes='boo', ) ]) db.update_used_query_range(name=f'{exchange_name}_trades', start_ts=0, end_ts=9999) db.update_used_query_range(name=f'{exchange_name}_margins', start_ts=0, end_ts=9999) db.update_used_query_range(name=f'{exchange_name}_asset_movements', start_ts=0, end_ts=9999) # noqa: E501
def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> TRADES_LIST: """Queries trades for the given location and time range. If no location is given then all external, all exchange and DEX trades are queried. DEX Trades are queried only if the user has premium If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ trades: TRADES_LIST if location is not None: trades = self.query_location_trades(from_ts, to_ts, location) else: trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL) # crypto.com is not an API key supported exchange but user can import from CSV trades.extend( self.query_location_trades(from_ts, to_ts, Location.CRYPTOCOM)) for name, exchange in self.exchange_manager.connected_exchanges.items( ): exchange_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts) if self.premium is None: trades = self._apply_actions_limit( location=deserialize_location(name), action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # for all trades we also need uniswap trades if self.premium is not None: uniswap = self.chain_manager.uniswap if uniswap is not None: trades.extend( uniswap.get_trades( addresses=self.chain_manager. queried_addresses_for_module('uniswap'), from_timestamp=from_ts, to_timestamp=to_ts, ), ) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades
def _deserialize( self, value: str, attr: Optional[str], # pylint: disable=unused-argument data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument **_kwargs: Any, ) -> Location: try: location = deserialize_location(value) except DeserializationError as e: raise ValidationError(str(e)) return location
def _deserialize( self, value: str, attr, # pylint: disable=unused-argument data, # pylint: disable=unused-argument **kwargs, # pylint: disable=unused-argument ) -> Location: try: location = deserialize_location(value) except DeserializationError as e: raise ValidationError(str(e)) return location
def query_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, only_cache: bool, ) -> List[Trade]: """Queries the local DB and the remote exchange for the trade history of the user Limits the query to the given time range and also if only_cache is True returns only what is already saved in the DB without performing an exchange query """ trades = self.db.get_trades( from_ts=start_ts, to_ts=end_ts, location=deserialize_location(self.name), ) if only_cache: return trades ranges = DBQueryRanges(self.db) ranges_to_query = ranges.get_location_query_ranges( location_string=f'{self.name}_trades', start_ts=start_ts, end_ts=end_ts, ) new_trades = [] for query_start_ts, query_end_ts in ranges_to_query: # If we have a time frame we have not asked the exchange for trades then # go ahead and do that now new_trades.extend( self.query_online_trade_history( start_ts=query_start_ts, end_ts=query_end_ts, )) # make sure to add them to the DB if new_trades != []: self.db.add_trades(new_trades) # and also set the used queried timestamp range for the exchange ranges.update_used_query_range( location_string=f'{self.name}_trades', start_ts=start_ts, end_ts=end_ts, ranges_to_query=ranges_to_query, ) # finally append them to the already returned DB trades trades.extend(new_trades) return trades
def query_trade_history( self, start_ts: Timestamp, end_ts: Timestamp, ) -> List[Trade]: """Queries the local DB and the remote exchange for the trade history of the user""" trades = self.db.get_trades( from_ts=start_ts, to_ts=end_ts, location=deserialize_location(self.name), ) ranges = DBQueryRanges(self.db) ranges_to_query = ranges.get_location_query_ranges( location_string=f'{self.name}_trades', start_ts=start_ts, end_ts=end_ts, ) new_trades = [] for query_start_ts, query_end_ts in ranges_to_query: # If we have a time frame we have not asked the exchange for trades then # go ahead and do that now try: new_trades.extend( self.query_online_trade_history( start_ts=query_start_ts, end_ts=query_end_ts, )) except NotImplementedError: msg = 'query_online_trade_history should only not be implemented by bitmex' assert self.name == 'bitmex', msg # make sure to add them to the DB if new_trades != []: self.db.add_trades(new_trades) # and also set the used queried timestamp range for the exchange ranges.update_used_query_range( location_string=f'{self.name}_trades', start_ts=start_ts, end_ts=end_ts, ranges_to_query=ranges_to_query, ) # finally append them to the already returned DB trades trades.extend(new_trades) return trades
def _query_exchange_asset_movements( self, from_ts: Timestamp, to_ts: Timestamp, all_movements: List[AssetMovement], exchange: Union[ExchangeInterface, Location], only_cache: bool, ) -> List[AssetMovement]: if isinstance(exchange, ExchangeInterface): location = deserialize_location(exchange.name) # clear the asset movements queried for this exchange self.actions_per_location['asset_movement'][location] = 0 location_movements = exchange.query_deposits_withdrawals( start_ts=from_ts, end_ts=to_ts, only_cache=only_cache, ) else: assert isinstance(exchange, Location), 'only a location should make it here' assert exchange == Location.CRYPTOCOM, 'only cryptocom should make it here' location = exchange # cryptocom has no exchange integration but we may have DB entries self.actions_per_location['asset_movement'][location] = 0 location_movements = self.data.db.get_asset_movements( from_ts=from_ts, to_ts=to_ts, location=location, ) movements: List[AssetMovement] = [] if self.premium is None: movements = self._apply_actions_limit( location=location, action_type='asset_movement', location_actions=location_movements, all_actions=all_movements, ) else: all_movements.extend(location_movements) movements = all_movements return movements
def check_saved_events_for_exchange( exchange_name: str, db: DBHandler, should_exist: bool, ) -> None: trades = db.get_trades(location=deserialize_location(exchange_name)) trades_range = db.get_used_query_range(f'{exchange_name}_trades') margins_range = db.get_used_query_range(f'{exchange_name}_margins') movements_range = db.get_used_query_range( f'{exchange_name}_asset_movements') if should_exist: assert trades_range is not None assert margins_range is not None assert movements_range is not None assert len(trades) != 0 else: assert trades_range is None assert margins_range is None assert movements_range is None assert len(trades) == 0
def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], ) -> List[Trade]: """Queries trades for the given location and time range. If no location is given then all external and all exchange trades are queried. If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connectingto any of the remote exchanges """ if location is not None: trades = self.query_location_trades(from_ts, to_ts, location) else: trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL) for name, exchange in self.exchange_manager.connected_exchanges.items( ): exchange_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts) if self.premium is None: trades = self._apply_actions_limit( location=deserialize_location(name), action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades
def create_action(self, index: int, ts: Timestamp): """Create a random trade action on a random exchange depending on the funds that are available in that exchange""" # choose an exchange at random exchange_name = random.choice(ALLOWED_EXCHANGES) exchange = getattr(self, exchange_name) # choose a random pair at that exchange pair = exchange.choose_pair( timestamp=ts, price_query=self.query_historical_price, ) print( f'Creating trade {index + 1} / {self.trades_number} in {exchange_name}' f' for the pair: {pair} at timestamp {ts}', ) # depending on our funds decide on what to do. Buy/sell base, quote = pair_get_assets(pair) if exchange.get_balance(base) is None: action_type = TradeType.BUY elif exchange.get_balance(quote) is None: action_type = TradeType.SELL else: # TODO: trade the one we have most of action_type = random.choice(list(TradeType)) # if we are buying we are going to spend from the quote asset if action_type == TradeType.BUY: spending_asset = quote else: # selling spends from the base asset spending_asset = base # get a spending asset amount within our per-trade equivalent range and # our available funds spending_usd_rate = self.query_historical_price( spending_asset, A_USD, ts) max_usd_in_spending_asset = spending_usd_rate * exchange.get_balance( spending_asset) max_usd_equivalent_to_spend = min(max_usd_in_spending_asset, MAX_TRADE_USD_VALUE) rate = self.query_historical_price(base, quote, ts) usd_to_spend = FVal( random.uniform(0.01, float(max_usd_equivalent_to_spend))) amount_in_spending_asset = usd_to_spend / spending_usd_rate # if we are buying then the amount is the amount of asset we bought if action_type == TradeType.BUY: amount = amount_in_spending_asset / rate # if we are selling the amount is the spending asset amount else: amount = amount_in_spending_asset quote_asset_usd_rate = self.query_historical_price(quote, A_USD, ts) fee_in_quote_currency = FVal(random.uniform( 0, MAX_FEE_USD_VALUE)) / quote_asset_usd_rate # create the trade trade = Trade( timestamp=ts, location=deserialize_location(exchange_name), pair=pair, trade_type=action_type, amount=amount, rate=rate, fee=fee_in_quote_currency, fee_currency=quote, link='', notes='', ) logger.info(f'Created trade: {trade}') # Adjust our global and per exchange accounting if action_type == TradeType.BUY: # we buy so we increase our base asset by amount self.increase_asset(base, amount, exchange_name) # and decrease quote by amount * rate self.decrease_asset(quote, amount * rate, exchange_name) else: # we sell so we increase our quote asset self.increase_asset(quote, amount * rate, exchange_name) # and decrease our base asset self.decrease_asset(base, amount, exchange_name) # finally add it to the exchange exchange.append_trade(trade)
def create_fake_data(self, args: argparse.Namespace) -> None: self._clean_tables() from_ts, to_ts = StatisticsFaker._get_timestamps(args) starting_amount, min_amount, max_amount = StatisticsFaker._get_amounts( args) total_amount = starting_amount locations = [ deserialize_location(location) for location in args.locations.split(',') ] assets = [Asset(symbol) for symbol in args.assets.split(',')] go_up_probability = FVal(args.go_up_probability) # Add the first distribution of location data location_data = [] for idx, value in enumerate( divide_number_in_parts(starting_amount, len(locations))): location_data.append( LocationData( time=from_ts, location=locations[idx].serialize_for_db(), usd_value=str(value), )) # add the location data + total to the DB self.db.add_multiple_location_data(location_data + [ LocationData( time=from_ts, location=Location.TOTAL.serialize_for_db(), usd_value=str(total_amount), ) ]) # Add the first distribution of assets assets_data = [] for idx, value in enumerate( divide_number_in_parts(starting_amount, len(assets))): assets_data.append( DBAssetBalance( category=BalanceType.ASSET, time=from_ts, asset=assets[idx], amount=str(random.randint(1, 20)), usd_value=str(value), )) self.db.add_multiple_balances(assets_data) while from_ts < to_ts: print( f'At timestamp: {from_ts}/{to_ts} wih total net worth: ${total_amount}' ) new_location_data = [] new_assets_data = [] from_ts += args.seconds_between_balance_save # remaining_loops = to_ts - from_ts / args.seconds_between_balance_save add_usd_value = random.choice([100, 350, 500, 625, 725, 915, 1000]) add_amount = random.choice([ FVal('0.1'), FVal('0.23'), FVal('0.34'), FVal('0.69'), FVal('1.85'), FVal('2.54'), ]) go_up = ( # If any asset's usd value is close to to go below zero, go up any( FVal(a.usd_value) - FVal(add_usd_value) < 0 for a in assets_data) or # If total is going under the min amount go up total_amount - add_usd_value < min_amount or # If "dice roll" matched and we won't go over the max amount go up (add_usd_value + total_amount < max_amount and FVal(random.random()) <= go_up_probability)) if go_up: total_amount += add_usd_value action = operator.add else: total_amount -= add_usd_value action = operator.sub for idx, value in enumerate( divide_number_in_parts(add_usd_value, len(locations))): new_location_data.append( LocationData( time=from_ts, location=location_data[idx].location, usd_value=str( action(FVal(location_data[idx].usd_value), value)), )) # add the location data + total to the DB self.db.add_multiple_location_data(new_location_data + [ LocationData( time=from_ts, location=Location.TOTAL.serialize_for_db(), usd_value=str(total_amount), ) ]) for idx, value in enumerate( divide_number_in_parts(add_usd_value, len(assets))): old_amount = FVal(assets_data[idx].amount) new_amount = action(old_amount, add_amount) if new_amount < FVal('0'): new_amount = old_amount + FVal('0.01') new_assets_data.append( DBAssetBalance( category=BalanceType.ASSET, time=from_ts, asset=assets[idx], amount=str(new_amount), usd_value=str( action(FVal(assets_data[idx].usd_value), value)), )) self.db.add_multiple_balances(new_assets_data) location_data = new_location_data assets_data = new_assets_data
def query_trades( self, from_ts: Timestamp, to_ts: Timestamp, location: Optional[Location], only_cache: bool, ) -> TRADES_LIST: """Queries trades for the given location and time range. If no location is given then all external, all exchange and DEX trades are queried. If only_cache is given then only trades cached in the DB are returned. No service is queried. DEX Trades are queried only if the user has premium If the user does not have premium then a trade limit is applied. May raise: - RemoteError: If there are problems connecting to any of the remote exchanges """ trades: TRADES_LIST if location is not None: trades = self.query_location_trades(from_ts, to_ts, location, only_cache) else: trades = self.query_location_trades(from_ts, to_ts, Location.EXTERNAL, only_cache) # crypto.com is not an API key supported exchange but user can import from CSV trades.extend( self.query_location_trades( from_ts=from_ts, to_ts=to_ts, location=Location.CRYPTOCOM, only_cache=only_cache, )) for name, exchange in self.exchange_manager.connected_exchanges.items( ): exchange_trades = exchange.query_trade_history( start_ts=from_ts, end_ts=to_ts, only_cache=only_cache, ) if self.premium is None: trades = self._apply_actions_limit( location=deserialize_location(name), action_type='trade', location_actions=exchange_trades, all_actions=trades, ) else: trades.extend(exchange_trades) # for all trades we also need the trades from the amm protocols if self.premium is not None: for amm_location in AMMTradeLocations: amm_module_name = cast(AMMTRADE_LOCATION_NAMES, str(amm_location)) amm_module = self.chain_manager.get_module(amm_module_name) if amm_module is not None: trades.extend( amm_module.get_trades( addresses=self.chain_manager. queried_addresses_for_module( amm_module_name), # noqa: E501 from_timestamp=from_ts, to_timestamp=to_ts, only_cache=only_cache, ), ) # return trades with most recent first trades.sort(key=lambda x: x.timestamp, reverse=True) return trades