def test_add_remove_exchange(user_data_dir):
"""Tests that adding and removing an exchange in the DB works.
Also unknown exchanges should fail
"""
msg_aggregator = MessagesAggregator()
db = DBHandler(user_data_dir, '123', msg_aggregator, None)
# Test that an unknown exchange fails
with pytest.raises(InputError):
db.add_exchange('non_existing_exchange', 'api_key', 'api_secret')
credentials = db.get_exchange_credentials()
assert len(credentials) == 0
kraken_api_key = ApiKey('kraken_api_key')
kraken_api_secret = ApiSecret(b'kraken_api_secret')
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
# add mock kraken and binance
db.add_exchange('kraken', kraken_api_key, kraken_api_secret)
db.add_exchange('binance', binance_api_key, binance_api_secret)
# and check the credentials can be retrieved
credentials = db.get_exchange_credentials()
assert len(credentials) == 2
assert credentials['kraken'].api_key == kraken_api_key
assert credentials['kraken'].api_secret == kraken_api_secret
assert credentials['binance'].api_key == binance_api_key
assert credentials['binance'].api_secret == binance_api_secret
# remove an exchange and see it works
db.remove_exchange('kraken')
credentials = db.get_exchange_credentials()
assert len(credentials) == 1
def test_binance_query_trade_history_custom_markets(function_scope_binance,
user_data_dir):
"""Test that custom pairs are queried correctly"""
msg_aggregator = MessagesAggregator()
db = DBHandler(user_data_dir, '123', msg_aggregator, None)
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
db.add_exchange('binance', Location.BINANCE, binance_api_key,
binance_api_secret)
binance = function_scope_binance
markets = ['ETHBTC', 'BNBBTC', 'BTCUSDC']
db.set_binance_pairs('binance', markets, Location.BINANCE)
count = 0
p = re.compile(r'symbol=[A-Z]*')
seen = set()
def mock_my_trades(url, timeout): # pylint: disable=unused-argument
nonlocal count
count += 1
market = p.search(url).group()[7:]
assert market in markets and market not in seen
seen.add(market)
text = '[]'
return MockResponse(200, text)
with patch.object(binance.session, 'get', side_effect=mock_my_trades):
binance.query_trade_history(start_ts=0,
end_ts=1564301134,
only_cache=False)
assert count == len(markets)
def maybe_include_etherscan_key(db: DBHandler, include_etherscan_key: bool) -> None:
if not include_etherscan_key:
return
# Add the tests only etherscan API key
db.add_external_service_credentials([ExternalServiceApiCredentials(
service=ExternalService.ETHERSCAN,
api_key=ApiKey('8JT7WQBB2VQP5C3416Y8X3S8GBA3CVZKP4'),
)])
def __init__(self, args: argparse.Namespace) -> None:
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.create_new_user(args.user_name, args.user_password)
# Start the fake exchanges API for the duration of the fake
# history creation. We need it up so that we can emulate responses
# from the exchanges
self.fake_kraken = FakeKraken()
self.fake_binance = FakeBinance()
mock_api = RestAPI(fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance)
self.mock_server = APIServer(rest_api=mock_api)
self.mock_server.start()
self.rotki.setup_exchange(
name='kraken',
location=Location.KRAKEN,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.rotki.setup_exchange(
name='binance',
location=Location.BINANCE,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.writer = ActionWriter(
trades_number=args.trades_number,
seconds_between_trades=args.seconds_between_trades,
seconds_between_balance_save=args.seconds_between_balance_save,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance,
)
self.writer.generate_history()
# stop the fake exchange API. Will be started again once we are finished,
# ready to serve the Rotkehlchen client
self.mock_server.stop()
def _deserialize(
self,
value: str,
attr: Optional[str], # pylint: disable=unused-argument
data: Optional[Mapping[str, Any]], # pylint: disable=unused-argument
**_kwargs: Any,
) -> ApiKey:
if not isinstance(value, str):
raise ValidationError('Given API Key should be a string')
return ApiKey(value)
def _deserialize(
self,
value: str,
attr, # pylint: disable=unused-argument
data, # pylint: disable=unused-argument
**kwargs, # pylint: disable=unused-argument
) -> ApiKey:
if not isinstance(value, str):
raise ValidationError('Given API Key should be a string')
return ApiKey(value)
def create_test_bitmex(
database: DBHandler,
msg_aggregator: MessagesAggregator,
) -> Bitmex:
# API key/secret from tests cases here: https://www.bitmex.com/app/apiKeysUsage
bitmex = Bitmex(
api_key=ApiKey('LAqUlngMIQkIUjXMUreyu3qn'),
secret=ApiSecret(b'chNOOS4KvNXR_Xq4k4c9qsfoKWvnDecLATCRlcBwyKDYnWgO'),
database=database,
msg_aggregator=msg_aggregator,
)
bitmex.first_connection_made = True
return bitmex
def maybe_include_cryptocompare_key(db: DBHandler, include_cryptocompare_key: bool) -> None:
if not include_cryptocompare_key:
return
keys = [
'a4a36d7fd1835cc1d757186de8e7357b4478b73923933d09d3689140ecc23c03',
'e929bcf68fa28715fa95f3bfa3baa3b9a6bc8f12112835586c705ab038ee06aa',
'5159ca00f2579ef634b7f210ad725550572afbfb44e409460dd8a908d1c6416a',
'6781b638eca6c3ca51a87efcdf0b9032397379a0810c5f8198a25493161c318d',
]
# Add the tests only etherscan API key
db.add_external_service_credentials([ExternalServiceApiCredentials(
service=ExternalService.CRYPTOCOMPARE,
api_key=ApiKey(random.choice(keys)),
)])
def test_binance_pairs(user_data_dir):
msg_aggregator = MessagesAggregator()
db = DBHandler(user_data_dir, '123', msg_aggregator, None)
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
db.add_exchange('binance', Location.BINANCE, binance_api_key,
binance_api_secret)
db.set_binance_pairs('binance', ['ETHUSDC', 'ETHBTC', 'BNBBTC'],
Location.BINANCE)
query = db.get_binance_pairs('binance', Location.BINANCE)
assert query == ['ETHUSDC', 'ETHBTC', 'BNBBTC']
db.set_binance_pairs('binance', [], Location.BINANCE)
query = db.get_binance_pairs('binance', Location.BINANCE)
assert query == []
def create_patched_premium_with_keypair(
patch_get: bool,
metadata_last_modify_ts=None,
metadata_data_hash=None,
saved_data=None,
):
api_key = ApiKey(base64.b64encode(make_random_b64bytes(128)))
api_secret = ApiSecret(base64.b64encode(make_random_b64bytes(128)))
patches = patched_create_premium(
api_key,
api_secret,
patch_get,
metadata_last_modify_ts,
metadata_data_hash,
saved_data,
)
return api_key, api_secret, patches[0], patches[1]
def _init_database(
data_dir: FilePath,
password: str,
msg_aggregator: MessagesAggregator,
db_settings: Optional[Dict[str, Any]],
ignored_assets: Optional[List[Asset]],
blockchain_accounts: BlockchainAccounts,
include_etherscan_key: bool,
) -> DBHandler:
db = DBHandler(data_dir, password, msg_aggregator)
settings = {
# DO not submit usage analytics during tests
'submit_usage_analytics': False,
'main_currency': DEFAULT_TESTS_MAIN_CURRENCY,
}
# Set the given db_settings. The pre-set values have priority unless overriden here
if db_settings is not None:
for key, value in db_settings.items():
settings[key] = value
db.set_settings(ModifiableDBSettings(**settings))
if ignored_assets:
for asset in ignored_assets:
db.add_to_ignored_assets(asset)
# Make sure that the fixture provided accounts are in the blockchain
db.add_blockchain_accounts(
SupportedBlockchain.ETHEREUM,
[BlockchainAccountData(address=x) for x in blockchain_accounts.eth],
)
db.add_blockchain_accounts(
SupportedBlockchain.BITCOIN,
[BlockchainAccountData(address=x) for x in blockchain_accounts.btc],
)
if include_etherscan_key:
# Add the tests only etherscan API key
db.add_external_service_credentials([
ExternalServiceApiCredentials(
service=ExternalService.ETHERSCAN,
api_key=ApiKey('8JT7WQBB2VQP5C3416Y8X3S8GBA3CVZKP4'),
)
])
return db
def test_add_remove_exchange(user_data_dir):
"""Tests that adding and removing an exchange in the DB works.
Also unknown exchanges should fail
"""
msg_aggregator = MessagesAggregator()
db = DBHandler(user_data_dir, '123', msg_aggregator, None)
# Test that an unknown exchange fails
with pytest.raises(InputError):
db.add_exchange('foo', Location.EXTERNAL, 'api_key', 'api_secret')
credentials = db.get_exchange_credentials()
assert len(credentials) == 0
kraken_api_key1 = ApiKey('kraken_api_key')
kraken_api_secret1 = ApiSecret(b'kraken_api_secret')
kraken_api_key2 = ApiKey('kraken_api_key2')
kraken_api_secret2 = ApiSecret(b'kraken_api_secret2')
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
# add mock kraken and binance
db.add_exchange('kraken1', Location.KRAKEN, kraken_api_key1,
kraken_api_secret1)
db.add_exchange('kraken2', Location.KRAKEN, kraken_api_key2,
kraken_api_secret2)
db.add_exchange('binance', Location.BINANCE, binance_api_key,
binance_api_secret)
# and check the credentials can be retrieved
credentials = db.get_exchange_credentials()
assert len(credentials) == 2
assert len(credentials[Location.KRAKEN]) == 2
kraken1 = credentials[Location.KRAKEN][0]
assert kraken1.name == 'kraken1'
assert kraken1.api_key == kraken_api_key1
assert kraken1.api_secret == kraken_api_secret1
kraken2 = credentials[Location.KRAKEN][1]
assert kraken2.name == 'kraken2'
assert kraken2.api_key == kraken_api_key2
assert kraken2.api_secret == kraken_api_secret2
assert len(credentials[Location.BINANCE]) == 1
binance = credentials[Location.BINANCE][0]
assert binance.name == 'binance'
assert binance.api_key == binance_api_key
assert binance.api_secret == binance_api_secret
# remove an exchange and see it works
db.remove_exchange('kraken1', Location.KRAKEN)
credentials = db.get_exchange_credentials()
assert len(credentials) == 2
assert len(credentials[Location.KRAKEN]) == 1
kraken2 = credentials[Location.KRAKEN][0]
assert kraken2.name == 'kraken2'
assert kraken2.api_key == kraken_api_key2
assert kraken2.api_secret == kraken_api_secret2
assert len(credentials[Location.BINANCE]) == 1
binance = credentials[Location.BINANCE][0]
assert binance.name == 'binance'
assert binance.api_key == binance_api_key
assert binance.api_secret == binance_api_secret
# remove last exchange of a location and see nothing is returned
db.remove_exchange('kraken2', Location.KRAKEN)
credentials = db.get_exchange_credentials()
assert len(credentials) == 1
assert len(credentials[Location.BINANCE]) == 1
binance = credentials[Location.BINANCE][0]
assert binance.name == 'binance'
assert binance.api_key == binance_api_key
assert binance.api_secret == binance_api_secret
def make_api_key() -> ApiKey:
return ApiKey(make_random_b64bytes(128).decode())
def test_associated_locations(database):
"""Test that locations imported in different places are correctly stored in database"""
# Add trades from different locations
trades = [Trade(
timestamp=Timestamp(1595833195),
location=Location.CRYPTOCOM,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1587825824),
location=Location.CRYPTOCOM,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596014214),
location=Location.BLOCKFI,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1565888464),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596014214),
location=Location.NEXO,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('50.0')),
rate=Price(FVal('3.521')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1612051199),
location=Location.BLOCKFI,
base_asset=symbol_to_asset_or_token('USDC'),
quote_asset=symbol_to_asset_or_token('LTC'),
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('6404.6')),
rate=Price(FVal('151.6283999982779809352223797')),
fee=None,
fee_currency=None,
link='',
notes='One Time',
), Trade(
timestamp=Timestamp(1595833195),
location=Location.POLONIEX,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1.0')),
rate=Price(FVal('281.14')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596429934),
location=Location.COINBASE,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('0.00061475')),
rate=Price(FVal('309.0687271248474989833265555')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
), Trade(
timestamp=Timestamp(1596429934),
location=Location.EXTERNAL,
base_asset=A_ETH,
quote_asset=A_EUR,
trade_type=TradeType.BUY,
amount=AssetAmount(FVal('1')),
rate=Price(FVal('320')),
fee=Fee(ZERO),
fee_currency=A_USD,
link='',
notes='',
)]
# Add multiple entries for same exchange + connected exchange
database.add_trades(trades)
kraken_api_key1 = ApiKey('kraken_api_key')
kraken_api_secret1 = ApiSecret(b'kraken_api_secret')
kraken_api_key2 = ApiKey('kraken_api_key2')
kraken_api_secret2 = ApiSecret(b'kraken_api_secret2')
binance_api_key = ApiKey('binance_api_key')
binance_api_secret = ApiSecret(b'binance_api_secret')
# add mock kraken and binance
database.add_exchange('kraken1', Location.KRAKEN, kraken_api_key1, kraken_api_secret1)
database.add_exchange('kraken2', Location.KRAKEN, kraken_api_key2, kraken_api_secret2)
database.add_exchange('binance', Location.BINANCE, binance_api_key, binance_api_secret)
# Add uniswap and sushiswap events
database.add_amm_events([
LiquidityPoolEvent(
tx_hash='0x47ea26957ce09e84a51b51dfdab6a4ac1c3672a372eef77b15ef7677174ac847',
log_index=23,
address=ChecksumEthAddress('0x3163Bb273E8D9960Ce003fD542bF26b4C529f515'),
timestamp=Timestamp(1590011534),
event_type=EventType.MINT_SUSHISWAP,
pool_address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
token0=EthereumToken('0x514910771AF9Ca656af840dff83E8264EcF986CA'),
token1=EthereumToken('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'),
amount0=FVal('3.313676003468974932'),
amount1=FVal('0.064189269269768657'),
usd_price=FVal('26.94433946158740371839009166230438'),
lp_amount=FVal('0.460858304063739927'),
),
])
database.add_amm_swaps([
AMMSwap(
tx_hash='0xa54bf4c68d435e3c8f432fd7e62b7f8aca497a831a3d3fca305a954484ddd7b2',
log_index=208,
address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
from_address=string_to_ethereum_address('0xd9e1cE17f2641f24aE83637ab66a2cca9C378B9F'),
to_address=string_to_ethereum_address('0xC9cB53B48A2f3A9e75982685644c1870F1405CCb'),
timestamp=Timestamp(1609301469),
location=Location.UNISWAP,
token0=EthereumToken('0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2'),
token1=EthereumToken('0xdAC17F958D2ee523a2206206994597C13D831ec7'),
amount0_in=AssetAmount(FVal('2.6455727132446468')),
amount1_in=AssetAmount(ZERO),
amount0_out=AssetAmount(ZERO),
amount1_out=AssetAmount(FVal('1936.810111')),
),
])
database.add_balancer_events([
BalancerEvent(
tx_hash='0xa54bf4c68d435e3c8f432fd7e62b7f8aca497a831a3d3fca305a954484ddd7b3',
log_index=23,
address=ChecksumEthAddress('0xa2107FA5B38d9bbd2C461D6EDf11B11A50F6b974'),
timestamp=Timestamp(1609301469),
event_type=BalancerBPTEventType.MINT,
pool_address_token=EthereumToken('0x514910771AF9Ca656af840dff83E8264EcF986CA'),
lp_balance=Balance(amount=FVal(2), usd_value=FVal(3)),
amounts=[
AssetAmount(FVal(1)),
AssetAmount(FVal(2)),
],
),
])
expected_locations = {
Location.KRAKEN,
Location.BINANCE,
Location.BLOCKFI,
Location.NEXO,
Location.CRYPTOCOM,
Location.POLONIEX,
Location.COINBASE,
Location.EXTERNAL,
Location.SUSHISWAP,
Location.UNISWAP,
Location.BALANCER,
}
assert set(database.get_associated_locations()) == expected_locations