def test_upload_data_to_server_same_hash(rotkehlchen_instance, db_password):
"""Test that if the server has same data hash as we no upload happens"""
rotkehlchen_instance.premium = Premium(
api_key=base64.b64encode(make_random_b64bytes(128)),
api_secret=base64.b64encode(make_random_b64bytes(128)),
)
last_ts = rotkehlchen_instance.data.db.get_last_data_upload_ts()
assert last_ts == 0
# Write anything in the DB to set a non-zero last_write_ts
rotkehlchen_instance.data.db.set_main_currency('EUR')
_, our_hash = rotkehlchen_instance.data.compress_and_encrypt_db(
db_password)
patched_put = patch.object(
rotkehlchen_instance.premium.session,
'put',
return_value=None,
)
patched_get = create_patched_premium_session_get(
session=rotkehlchen_instance.premium.session,
metadata_last_modify_ts=0,
metadata_data_hash=our_hash,
saved_data='foo',
)
with patched_get, patched_put as put_mock:
rotkehlchen_instance.upload_data_to_server()
# The upload mock should not have been called since the hash is the same
assert not put_mock.called
def __init__(self, args):
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.faker = Faker()
self.create_new_user(args.user_name, args.user_password)
# register the exchanges with the rotkehlchen DB of the new user
self.rotki.data.db.add_exchange(
name='kraken',
api_key=make_random_b64bytes(128),
api_secret=make_random_b64bytes(128),
)
self.fake_kraken = FakeKraken()
self.writer = ActionWriter(
trades_number=args.trades_number,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
)
self.writer.generate_history()
def kraken(session_data_dir):
mock = MockKraken(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=session_data_dir,
)
return mock
def poloniex(session_data_dir, session_inquirer, messages_aggregator):
mock = MockPoloniex(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=session_data_dir,
msg_aggregator=messages_aggregator,
)
return mock
def bittrex(session_data_dir, session_inquirer):
mock = MockBittrex(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=session_data_dir,
msg_aggregator=MessagesAggregator(),
)
return mock
def kraken(session_data_dir, session_inquirer):
mock = MockKraken(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=session_data_dir,
usd_eur_price=session_inquirer.query_fiat_pair(S_EUR, S_USD),
)
return mock
def function_scope_poloniex(accounting_data_dir, inquirer,
function_scope_messages_aggregator):
mock = MockPoloniex(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=accounting_data_dir,
msg_aggregator=function_scope_messages_aggregator,
)
return mock
def function_scope_kraken(accounting_data_dir, inquirer, function_scope_messages_aggregator):
mock = MockKraken(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
user_directory=accounting_data_dir,
msg_aggregator=function_scope_messages_aggregator,
usd_eur_price=inquirer.query_fiat_pair(S_EUR, S_USD),
)
return mock
def create_patched_premium_with_keypair(
patch_get: bool,
metadata_last_modify_ts=None,
metadata_data_hash=None,
saved_data=None,
):
api_key = ApiKey(base64.b64encode(make_random_b64bytes(128)))
api_secret = ApiSecret(base64.b64encode(make_random_b64bytes(128)))
patches = patched_create_premium(
api_key,
api_secret,
patch_get,
metadata_last_modify_ts,
metadata_data_hash,
saved_data,
)
return api_key, api_secret, patches[0], patches[1]
def mock_binance(accounting_data_dir, inquirer): # pylint: disable=unused-argument
binance = Binance(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
data_dir=accounting_data_dir,
msg_aggregator=MessagesAggregator(),
)
this_dir = os.path.dirname(os.path.abspath(__file__))
json_path = Path(
this_dir
).parent / 'tests' / 'utils' / 'data' / 'binance_exchange_info.json'
with json_path.open('r') as f:
json_data = json.loads(f.read())
binance._symbols_to_pair = create_binance_symbols_to_pair(json_data)
binance.first_connection_made = True
return binance
def __init__(self, args: argparse.Namespace) -> None:
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.create_new_user(args.user_name, args.user_password)
# Start the fake exchanges API for the duration of the fake
# history creation. We need it up so that we can emulate responses
# from the exchanges
self.fake_kraken = FakeKraken()
self.fake_binance = FakeBinance()
mock_api = RestAPI(fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance)
self.mock_server = APIServer(rest_api=mock_api)
self.mock_server.start()
self.rotki.setup_exchange(
name='kraken',
location=Location.KRAKEN,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.rotki.setup_exchange(
name='binance',
location=Location.BINANCE,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.writer = ActionWriter(
trades_number=args.trades_number,
seconds_between_trades=args.seconds_between_trades,
seconds_between_balance_save=args.seconds_between_balance_save,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance,
)
self.writer.generate_history()
# stop the fake exchange API. Will be started again once we are finished,
# ready to serve the Rotkehlchen client
self.mock_server.stop()
def test_binance_assets_are_known(
accounting_data_dir,
inquirer, # pylint: disable=unused-argument
):
# use a real binance instance so that we always get the latest data
binance = Binance(
api_key=base64.b64encode(make_random_b64bytes(128)),
secret=base64.b64encode(make_random_b64bytes(128)),
data_dir=accounting_data_dir,
msg_aggregator=MessagesAggregator(),
)
mapping = binance.symbols_to_pair
binance_assets = set()
for _, pair in mapping.items():
binance_assets.add(pair.binance_base_asset)
binance_assets.add(pair.binance_quote_asset)
sorted_assets = sorted(binance_assets)
for binance_asset in sorted_assets:
_ = asset_from_binance(binance_asset)
def fixture_rotki_premium_credentials() -> PremiumCredentials:
return PremiumCredentials(
given_api_key=base64.b64encode(make_random_b64bytes(128)).decode(),
given_api_secret=base64.b64encode(make_random_b64bytes(128)).decode(),
)
from http import HTTPStatus
from typing import Optional
from unittest.mock import patch
from typing_extensions import Literal
from rotkehlchen.constants import ROTKEHLCHEN_SERVER_TIMEOUT
from rotkehlchen.premium.premium import Premium, PremiumCredentials
from rotkehlchen.rotkehlchen import Rotkehlchen
from rotkehlchen.tests.utils.constants import A_GBP, DEFAULT_TESTS_MAIN_CURRENCY
from rotkehlchen.tests.utils.factories import make_random_b64bytes
from rotkehlchen.tests.utils.mock import MockResponse
# Probably invalid remote data but bigger than the test local DB
INVALID_BUT_BIGGER_REMOTE_DATA = base64.b64encode(
make_random_b64bytes(16048)).decode()
# Remote data containing a different main currency (GBP)
REMOTE_DATA = b'faIQsH8WVnRtaDinjp7wyM22/Q7uM8scZ5C/ZEZZFpS/AUb10oc+HjDuyD5sQKkwTBmlfM4SSjooCXesURz2i1/YyhYjWK2YIECSdPhjUYrdvzNWLFmTtCu+lEw7urhYkC+22dpld3CO94ZM3xXr7WHzul+QEO9mT+snPGfKLAcSJhBSK/GXFi37Apgiz7u7LbgJ/Wp1D5C8ibYDIr83c39kaKvlDH6C+p7g9ncmy5meaa4vePxJlZZV6ay3x+yjZdffTdGm0RNl8OFzBNGs+eREjqk+vfWul5PgfHLdztaDGuYyhiw2+Hcos59BNpv3SHz9rBe6vcE7BZYPA+9zS77+qcQEyy9GEGXgCPX2UJpKNDR3ykHZtY+e2T+koBnzcspqEb9cDRe8pURWXcgNPh2+VSz4tDHF+GFEqbostAyZhkwjlB39QiJkK/HVymfLWtWL+/lQcvIrMlK6IVWmuEDCE74q0/RQX+uh6r3O+OTR5x+9FZEB3RcOhmx8OuLRCLptUL+KDqpINLeaM2Kl0TQjRj9Ot/8Q2iMe0wvzEKPj0c/8zJ8gJbEgrNS5xgWS8wtgiBdGlFYq/iopVcd8Gr1oZkK9RNPCKGXArwZ6cakMmf9nYceLOjaaw6FIV9445aRuc2n67cR96/1+697HC+x012va6akjf6ExHPjricKqlOfJhZebXGiPwXwH4m8S2aFSx7yCB8KzLNGWhgm1r7e0o75lBaCi0WONVMxym+PhfwYvZwrhmGlVqCIfucHaXYvwFBMawFBHH8LThpOg2GbKK3nLU/JXejuAK6xVWLgWt8wAWNJk3ZMajLkEhtxYtQAvggwCyvc3JQCbdQo1oAQA0dfTfqAJ54QQ5NJ6KI2G7fJSEUOIZnV+dcwjLLe6axIgj6JabV5xyguVH6jGmyN6ncbMY7gproA+jSNeZFPw+RQTHccrlsjPtDfF4HLHxiNKISrded7FIR5VGSqO6Gn1ML4l0Fst8kVxZuUpFzsXtgdQ8/DBjBAmGrjs+iDwJbh24J1jwZzukYuGu+Pibj68YET4eon2uENjqBGDekbFoGOKpimAXw5mIkZ/m5JWeGuhSqHUk7jSLHP96MBGMgh92SPLcucECgq7jGx0uCpzM6fAwAdoAQSOzdL5GmklNz9EYb5vqULAT6J2Ipdxe5nzE0fHne214B8PuGC5+uPU7NJLWl/6ubMaPLK4wSjPmj03m74FDcPVw5Kf0EdMt+9bHnLFtQY5a0XlopYU7xfBGbxoEmG2ixVoUNJvZD6tDRBp8X+yhPVZIWXBqx9QrsHFlYg/OFOsuVzx8OfmmPyj8IKAZKFNPElMhKgyfS/I2etkTqLc08vE0pNb4q2JmOh6Dh1eelJfpKujdxcYIT6MPGQYP6r30PFLnGWt+aNEJRMMFpK/R/iRIqYX6vMcgJHacOOz7QiPOcm9zutIOQx6vCqM1wFgxuR7Gd4lifwwf6VSyXK9W7RQ/9O8MQFoW4JEaLjcu5Axffw7rZpdG/WbQi6l+W5eX1m1ijPDz+/NkFbv6j6ZFUEwqJgegKDc5KUP7KvBk5T7F8EoWgpxXWKZl2XWhv6DZudRPkj3xyY8yLu5gCcvy/oHTU6X7ILh7mgxX2dl0IgFuCbaGHq7eIXPOCzist55aBL3haBQegQEG3xTMzX4zK+v+CWi6cTr+iDqWyVWgy61ZnB5t5+HrQ9VHfJALQhb00Qh/Jj5Vt0VIvFfqXKGazEksSdLY4HzxmckJIPMpSiRCglKYwC2Tc/YXZs5IaobSaEGcHJhkdkcON2CJMOfA4YXylHh4fQMZEGXeR0SQnsgcydpEj1xzY5Ucz0su8mEUWNaGH/PwatuVIOWk59ICrTfy9iL2yqnUxmFIdixUZiHnIihlHGOoskdOsor7akrRc7Ed0ztHtJxmWm/MwjbwYiXWYAiMLngEKJb96usa01uslyAIwylKa7mRjlj6k+sZ4NtkRS4e4Kdg9rvK9DR7uFuP+iFcjE1Xk12ANWFJi6BM51dmySjnep95Ur91cJXKPD3PYezSDUmjj3MSWtVMrSnOmbQuORoDmEW9Evp/D+nRsbiz32Kt/PLskNhrYbSoUlToaVWdzc0fUosZZ6kH+CnF7GbFNZU2Huc0qQsxesDsb2WbrXlg0Bidj43q7Ha3ZYJxTFTj3xZpBMYR9lhVPjdPb4Hl2ibtzwWw87fN6EuvyAu+4Sfujn4zsMu4qXA5YA1gStAaqlmZFa+0jKOrswCu0lC7h6i9FgSTpemUTWaMsaksk5yVRoBUGPM4AQXapSTjHP/sz+5kHlUPsEvaa68Ko+MwtxPFlrwzL4o6fOuLHR96q5lznuHrz26FihWfdeALNAUP2JA/d5W2TsEpdPCObuQcSZicONk4fnXmISWdVQQs4E/Oq/aQESIL55kO6GHEvKGzIYr9gGX3z+xTJoJg1fLTsiAWoyp1pCLOGP3jdsQSAo9MQSzHd6V8l5MRjgtCtOP4f2SarSi+QBEySiL0Vm/sF7sb2jcQu8Kc/wcoE8XQAujkyXJLoNJreTPaUs852Q/PRhOuTomMtNCbnR+L3jWEIzsM4w+HbGTPJQWJot3tjtNk4Zb3CmT+DuBlCnFs1kKXtwDHKReiohtt/pLA0pw4qzyMCbb8eyz0om/ik/o+CGL7QfpoeQBg1/LCcVPERBALETSoK78eM04YOc8LR5FiksJDPjw85bA2a9rIp47rbN+jBR7lkhCsCHhGw3mToLU/rP8Z+95Habk6SfnN1KasWDxB4Zzfx3OXxHmQQNK+NepDm/T3UtBA4eR9o9x3USgpy7+joQJRAxM4+dql/r82ULtWdmGxCabWwlzszOJFZ2wC6jPBACngADRQz1zuwDSlXo4so/k/UARASknsdX2Mz0F2mp1WHym+05ZA3X9NxIwm7UCmxuEI6KTRZKsHuuPL6dpzbo07c5vTFfrCLO/Lql9y8U5gwFIi3Eu7dRwm5i2U5s4xzaOjbsRNkFtDJauG1RitQuZutfUEIQD1AgjdZ65FeL21SWQKvZZXGL7SXhTGiiDXzWoKelFjbI0VEM+QrdjYmXq0UbnI7JOR05qTzsGgOMJ6mhqHH4T6gDOjn6RQuA=' # noqa: E501
# Remote data containing an older DB version
REMOTE_DATA_OLDER_DB = b'qLQdzIvX6c8jJyucladYh2wZvpPbna/hmokjLu8NX64blb+lM/HxIGtCq5YH9TNUV51VHGMqMBhUeHqdtdOh1/VLg5q2NuzNmiCUroV0u97YMYM5dsGrKpy+J9d1Hbq33dlx8YcQxBsJEM2lSmLXiW8DQ/AfNJfT7twe6u+w1i9soFF7hbkafnrg2s7QGukB8D4CY1sIcZd2VRlMy7ATwtOF9ur8KDrKfVpZSQlTsWfyfiWyJmcVTmvPjqPAmZ0PEDlwqmNETe6yeRnkKgU0T2xTrTAJkawoGn41g0LnYi+ghTBPTboiLVTqASk/C71ofdEjN0gacy/9wNIBrq3cvfZBsrTpjzt88W2pnPHbLdfxrycToeGKNBASexs42uzBWOqa6BFPEiy7mSzKClLp4q+hiZtasyhnwMzUYvsIb25BvXBAPJQnjcBW+hzuiwQp+C3hynxTSPY1v2S80i3fqDK7BKY8VpPpjV+tC5B0pn6PsBETKZjB1pPKQ//m/I8HI0bWb+0fpVs4NbK9nFpRN6Capd8wJTzWtSp7vGbHOoaDAwtNtp61QI7eDsiMZGYXFy5jn8CmE+uWC4zDhLmoAUwAehuUSjv0v5RJGX/IAgWxoRMhAEra54bRwZ0vY1YRBS/Xf/AXp17BRzqE8NwSAUstgizOk7ryT3BQaTqybrt4y4omyw1VVpeisJROVK0fcFJFFH1zYUbbUB+0CBRq20y54faSSNNjc05pYHv456BBBIwpUwMS4M7yZz+HwP8b/OIq0LMr7d5SJdDjG9Ut1siZbaGRdyqv86WNTiSrlMmTASHi7+z+Z8CX9GnmEgVJna5mvvOhBC/zIpZiRLzwbYjdvrtw3N9X+NHzIaDGrAo1LtWh+eGmRHPKlb+CICOMj4TGvtGKlL/IfzBcrBfeTwkNSge2l4mOFG9l82ci4RZ7I4Yr6WUQJ+NU6DYQYKb5wMz+xTJmenHHaQxy0fsTulO5/RKfY8u1O9xT5kDtNc/R00CDheqcTS773NLDL4dqHEE/+lVxoVdFT/VvxzHrBKnI6M1UyJgDHu1BFIto2/z2wS0GjVXkBVFvMfQTYMZmb88RP/04F00kt3wqg/lrhAqr60BaC/FzIKG9lepDXXBAhHZyy+a1HYCkJlA43QoX3duu3fauViP+2RN306/tFw6HJvkRiCU7E3T9tLOHU508PLhcN8a5ON7aVyBtzdGO5i57j6Xm96di79IsfwStowS31kDix+B1mYeD8R1nvthWOKgL2KiAl/UpbXDPOuVBYubZ+V4/D8jxRCivM2ukME+SCIGzraR3EBqAdvjp3dLC1tomnawaEzAQYTUHbHndYatmIYnzEsTzFd8OWoX/gy0KGaZJ/mUGDTFBbkWIDE8=' # noqa: E501
# Valid format but not "real" premium api key and secret
VALID_PREMIUM_KEY = (
'kWT/MaPHwM2W1KUEl2aXtkKG6wJfMW9KxI7SSerI6/QzchC45/GebPV9xYZy7f+VKBeh5nDRBJBCYn7WofMO4Q=='
)
VALID_PREMIUM_SECRET = (
'TEF5dFFrOFcwSXNrM2p1aDdHZmlndFRoMTZQRWJhU2dacTdscUZSeHZTRmJLRm5ZaVRlV2NYU'
'llYR1lxMjlEdUtRdFptelpCYmlXSUZGRTVDNWx3NDNYbjIx')
def mock_query_last_metadata(last_modify_ts, data_hash, data_size):
def rotkehlchen_api_secret():
return base64.b64encode(make_random_b64bytes(128))
def test_upload_data_to_server(rotkehlchen_instance, username, db_password):
"""Test our side of uploading data to the server"""
rotkehlchen_instance.premium = Premium(
api_key=base64.b64encode(make_random_b64bytes(128)),
api_secret=base64.b64encode(make_random_b64bytes(128)),
)
last_ts = rotkehlchen_instance.data.db.get_last_data_upload_ts()
assert last_ts == 0
# Write anything in the DB to set a non-zero last_write_ts
rotkehlchen_instance.data.db.set_main_currency('EUR')
last_write_ts = rotkehlchen_instance.data.db.get_last_write_ts()
_, our_hash = rotkehlchen_instance.data.compress_and_encrypt_db(
db_password)
remote_hash = 'a' + our_hash[1:]
def mock_succesfull_upload_data_to_server(
url, # pylint: disable=unused-argument
data,
timeout,
):
# Can't compare data blobs as they are encrypted and as such can be
# different each time
assert 'data_blob' in data
assert data['original_hash'] == our_hash
assert data['last_modify_ts'] == last_write_ts
assert 'index' in data
assert len(data['data_blob']) == data['length']
assert 'nonce' in data
assert data['compression'] == 'zlib'
assert timeout == ROTKEHLCHEN_SERVER_TIMEOUT
return MockResponse(200, '{"success": true}')
patched_put = patch.object(
rotkehlchen_instance.premium.session,
'put',
side_effect=mock_succesfull_upload_data_to_server,
)
patched_get = create_patched_premium_session_get(
session=rotkehlchen_instance.premium.session,
metadata_last_modify_ts=0,
metadata_data_hash=remote_hash,
saved_data='foo',
)
with patched_get, patched_put:
rotkehlchen_instance.upload_data_to_server()
now = ts_now()
last_ts = rotkehlchen_instance.data.db.get_last_data_upload_ts()
msg = 'The last data upload timestamp should have been saved in the db as now'
assert last_ts >= now and last_ts - now < 50, msg
last_ts = rotkehlchen_instance.last_data_upload_ts
msg = 'The last data upload timestamp should also be in memory'
assert last_ts >= now and last_ts - now < 50, msg
# and now logout and login again and make sure that the last_data_upload_ts is correct
rotkehlchen_instance.logout()
rotkehlchen_instance.data.unlock(username, db_password, create_new=False)
assert last_ts == rotkehlchen_instance.last_data_upload_ts
assert last_ts == rotkehlchen_instance.data.db.get_last_data_upload_ts()