def __init__(self, args):
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.faker = Faker()
self.create_new_user(args.user_name, args.user_password)
# register the exchanges with the rotkehlchen DB of the new user
self.rotki.data.db.add_exchange(
name='kraken',
api_key=make_random_b64bytes(128),
api_secret=make_random_b64bytes(128),
)
self.fake_kraken = FakeKraken()
self.writer = ActionWriter(
trades_number=args.trades_number,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
)
self.writer.generate_history()
def __init__(self) -> None:
"""Initializes the backend server
May raise:
- SystemPermissionError due to the given args containing a datadir
that does not have the correct permissions
"""
arg_parser = app_args(
prog='rotki',
description=
('Rotki, the portfolio tracker and accounting tool that respects your privacy'
),
)
self.args = arg_parser.parse_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = gevent.event.Event()
domain_list = []
if self.args.api_cors:
if "," in self.args.api_cors:
for domain in self.args.api_cors.split(","):
domain_list.append(str(domain))
else:
domain_list.append(str(self.args.api_cors))
self.api_server = APIServer(
rest_api=RestAPI(rotkehlchen=self.rotkehlchen),
cors_domain_list=domain_list,
)
def __init__(self):
self.args = app_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
self.greenlets = [mainloop_greenlet]
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def __init__(self):
self.args = app_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
# Greenlets that will be waited for when we shutdown
self.waited_greenlets = [mainloop_greenlet]
# Greenlets that can be killed instead of waited for when we shutdown
self.killable_greenlets = []
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def rotkehlchen_instance(
cli_args,
username,
blockchain,
accountant,
start_with_logged_in_user,
start_with_valid_premium,
function_scope_messages_aggregator,
db_password,
rotkehlchen_api_key,
rotkehlchen_api_secret,
):
"""A partially mocked rotkehlchen instance"""
rotkehlchen = Rotkehlchen(cli_args)
initialize_mock_rotkehlchen_instance(
rotki=rotkehlchen,
start_with_logged_in_user=start_with_logged_in_user,
start_with_valid_premium=start_with_valid_premium,
msg_aggregator=function_scope_messages_aggregator,
username=username,
accountant=accountant,
blockchain=blockchain,
db_password=db_password,
rotkehlchen_api_key=rotkehlchen_api_key,
rotkehlchen_api_secret=rotkehlchen_api_secret,
)
return rotkehlchen
def __init__(self):
arg_parser = app_args(
prog='rotkehlchen',
description='Rotkehlchen Crypto Portfolio Management',
)
self.args = arg_parser.parse_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
# Greenlets that will be waited for when we shutdown
self.waited_greenlets = [mainloop_greenlet]
# Greenlets that can be killed instead of waited for when we shutdown
self.killable_greenlets = []
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def __init__(self, args: argparse.Namespace) -> None:
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.create_new_user(args.user_name, args.user_password)
# Start the fake exchanges API for the duration of the fake
# history creation. We need it up so that we can emulate responses
# from the exchanges
self.fake_kraken = FakeKraken()
self.fake_binance = FakeBinance()
mock_api = RestAPI(fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance)
self.mock_server = APIServer(rest_api=mock_api)
self.mock_server.start()
self.rotki.setup_exchange(
name='kraken',
location=Location.KRAKEN,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.rotki.setup_exchange(
name='binance',
location=Location.BINANCE,
api_key=ApiKey(str(make_random_b64bytes(128))),
api_secret=ApiSecret(make_random_b64bytes(128)),
)
self.writer = ActionWriter(
trades_number=args.trades_number,
seconds_between_trades=args.seconds_between_trades,
seconds_between_balance_save=args.seconds_between_balance_save,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance,
)
self.writer.generate_history()
# stop the fake exchange API. Will be started again once we are finished,
# ready to serve the Rotkehlchen client
self.mock_server.stop()
def uninitialized_rotkehlchen(cli_args, inquirer): # pylint: disable=unused-argument
"""A rotkehlchen instance that has only had __init__ run but is not unlocked
Adding the inquirer fixture as a requirement to make sure that any mocking that
happens at the inquirer level is reflected in the tests.
For this to happen inquirer fixture must be initialized before Rotkehlchen so
that the inquirer initialization in Rotkehlchen's __init__ uses the fixture's instance
"""
return Rotkehlchen(cli_args)
class DataFaker(object):
def __init__(self, args):
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.faker = Faker()
self.create_new_user(args.user_name, args.user_password)
# register the exchanges with the rotkehlchen DB of the new user
self.rotki.data.db.add_exchange(
name='kraken',
api_key=make_random_b64bytes(128),
api_secret=make_random_b64bytes(128),
)
self.fake_kraken = FakeKraken()
self.writer = ActionWriter(
trades_number=args.trades_number,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
)
self.writer.generate_history()
def create_new_user(self, user_name: Optional[str], given_password: str):
if not user_name:
user_name = self.faker.user_name()
logger.info(
f'Creating fake user {user_name} with password {given_password}')
self.rotki.unlock_user(
user=user_name,
password=given_password,
create_new=True,
sync_approval=False,
api_key='',
api_secret='',
)
class RotkehlchenServer():
def __init__(self) -> None:
"""Initializes the backend server
May raise:
- SystemPermissionError due to the given args containing a datadir
that does not have the correct permissions
"""
arg_parser = app_args(
prog='rotki',
description=
('Rotki, the portfolio tracker and accounting tool that respects your privacy'
),
)
self.args = arg_parser.parse_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = gevent.event.Event()
domain_list = []
if self.args.api_cors:
if "," in self.args.api_cors:
for domain in self.args.api_cors.split(","):
domain_list.append(str(domain))
else:
domain_list.append(str(self.args.api_cors))
self.api_server = APIServer(
rest_api=RestAPI(rotkehlchen=self.rotkehlchen),
cors_domain_list=domain_list,
)
def shutdown(self) -> None:
log.debug('Shutdown initiated')
self.api_server.stop()
self.stop_event.set()
def main(self) -> None:
if os.name != 'nt':
gevent.hub.signal(signal.SIGQUIT, self.shutdown)
gevent.hub.signal(signal.SIGINT, self.shutdown)
gevent.hub.signal(signal.SIGTERM, self.shutdown)
self.api_server.start(host=self.args.api_host, port=self.args.api_port)
# Start rotki's main loop
self.rotkehlchen.start()
self.stop_event.wait()
def __init__(self) -> None:
arg_parser = app_args(
prog='rotki',
description=
('Rotki, the portfolio tracker and accounting tool that respects your privacy'
),
)
self.args = arg_parser.parse_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = gevent.event.Event()
domain_list = []
if self.args.api_cors:
if "," in self.args.api_cors:
for domain in self.args.api_cors.split(","):
domain_list.append(str(domain))
else:
domain_list.append(str(self.args.api_cors))
self.api_server = APIServer(
rest_api=RestAPI(rotkehlchen=self.rotkehlchen),
cors_domain_list=domain_list,
)
class DataFaker(object):
def __init__(self, args: argparse.Namespace) -> None:
args.logfile = 'data_faker.log'
self.rotki = Rotkehlchen(args)
random_seed = datetime.datetime.now()
logger.info(f'Random seed used: {random_seed}')
random.seed(random_seed)
self.create_new_user(args.user_name, args.user_password)
# Start the fake exchanges API for the duration of the fake
# history creation. We need it up so that we can emulate responses
# from the exchanges
self.fake_kraken = FakeKraken()
self.fake_binance = FakeBinance()
mock_api = RestAPI(fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance)
self.mock_server = APIServer(rest_api=mock_api)
self.mock_server.start()
self.rotki.setup_exchange(
name='kraken',
api_key=str(make_random_b64bytes(128)),
api_secret=str(make_random_b64bytes(128)),
)
self.rotki.setup_exchange(
name='binance',
api_key=str(make_random_b64bytes(128)),
api_secret=str(make_random_b64bytes(128)),
)
self.writer = ActionWriter(
trades_number=args.trades_number,
seconds_between_trades=args.seconds_between_trades,
seconds_between_balance_save=args.seconds_between_balance_save,
rotkehlchen=self.rotki,
fake_kraken=self.fake_kraken,
fake_binance=self.fake_binance,
)
self.writer.generate_history()
# stop the fake exchange API. Will be started again once we are finished,
# ready to serve the Rotkehlchen client
self.mock_server.stop()
def create_new_user(self, user_name: Optional[str], given_password: str):
if not user_name:
user_name = ''.join(random.choices(ascii_uppercase + digits, k=10))
logger.info(
f'Creating fake user {user_name} with password {given_password}')
self.rotki.unlock_user(
user=user_name,
password=given_password,
create_new=True,
sync_approval='no',
premium_credentials=None,
)
def fixture_uninitialized_rotkehlchen(cli_args, inquirer, asset_resolver, globaldb): # pylint: disable=unused-argument # noqa: E501
"""A rotkehlchen instance that has only had __init__ run but is not unlocked
Adding the inquirer fixture as a requirement to make sure that any mocking that
happens at the inquirer level is reflected in the tests.
For this to happen inquirer fixture must be initialized before Rotkehlchen so
that the inquirer initialization in Rotkehlchen's __init__ uses the fixture's instance
Adding the AssetResolver as a requirement so that the first initialization happens here
"""
rotki = Rotkehlchen(cli_args)
return rotki
def test_initializing_rotki_with_datadir_with_wrong_permissions(
cli_args, data_dir):
os.chmod(data_dir, 0o200)
success = True
try:
with pytest.raises(SystemPermissionError):
Rotkehlchen(args=cli_args)
except Exception: # pylint: disable=broad-except
success = False
finally:
os.chmod(data_dir, 0o777)
assert success is True
def rotkehlchen_instance(cli_args, username, blockchain, accountant):
"""A partially mocked rotkehlchen instance"""
r = Rotkehlchen(cli_args)
r.data.unlock(username, '123', create_new=True)
# Remember accountant fixture has a mocked accounting data dir
# different to the usual user one
r.accountant = accountant
r.blockchain = blockchain
r.trades_historian = object()
r.user_is_logged_in = True
return r
def uninitialized_rotkehlchen(cli_args, inquirer, asset_resolver): # pylint: disable=unused-argument # noqa: E501
"""A rotkehlchen instance that has only had __init__ run but is not unlocked
Adding the inquirer fixture as a requirement to make sure that any mocking that
happens at the inquirer level is reflected in the tests.
For this to happen inquirer fixture must be initialized before Rotkehlchen so
that the inquirer initialization in Rotkehlchen's __init__ uses the fixture's instance
Adding the AssetResolver as a requirement so that the first initialization happens here
"""
# patch the constants to make sure that the periodic query for icons
# does not run during tests
size_patch = patch('rotkehlchen.rotkehlchen.ICONS_BATCH_SIZE', new=0)
sleep_patch = patch('rotkehlchen.rotkehlchen.ICONS_QUERY_SLEEP', new=999999)
with size_patch, sleep_patch:
rotki = Rotkehlchen(cli_args)
return rotki
class RotkehlchenServer(object):
def __init__(self):
self.args = app_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
self.greenlets = [mainloop_greenlet]
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def new_task_id(self):
with self.task_lock:
task_id = self.task_id
self.task_id += 1
return task_id
def write_task_result(self, task_id, result):
with self.task_lock:
self.task_results[task_id] = result
def get_task_result(self, task_id):
with self.task_lock:
return self.task_results[task_id]
def port(self):
return self.args.zerorpc_port
def shutdown(self):
logger.debug('Shutdown initiated')
self.zerorpc.stop()
gevent.wait(self.greenlets)
self.rotkehlchen.shutdown()
print("Shutting down zerorpc server")
logger.debug('Shutdown completed')
logging.shutdown()
def set_main_currency(self, currency_text):
self.rotkehlchen.set_main_currency(currency_text)
def set_settings(self, settings):
result, message = self.rotkehlchen.set_settings(settings)
return {'result': result, 'message': message}
def get_total_in_usd(self, balances):
total = 0
for _, entry in balances.items():
total += entry['usd_value']
return total
def handle_killed_greenlets(self, greenlet):
if greenlet.exception:
logger.error(
'Greenlet for task {} dies with exception: {}.\n'
'Exception Name: {}\nException Info: {}\nTraceback:\n {}'.
format(
greenlet.task_id,
greenlet.exception,
greenlet._exc_info[0],
greenlet._exc_info[1],
''.join(
traceback.format_tb(pickle.loads(
greenlet._exc_info[2]))),
))
# also write an error for the task result
result = {'error': str(greenlet.exception)}
self.write_task_result(greenlet.task_id, result)
def _query_async(self, command, task_id, **kwargs):
result = getattr(self, command)(**kwargs)
self.write_task_result(task_id, result)
def query_async(self, command, **kwargs):
task_id = self.new_task_id()
logger.debug("NEW TASK {} (kwargs:{}) with ID: {}".format(
command, kwargs, task_id))
greenlet = gevent.spawn(self._query_async, command, task_id, **kwargs)
greenlet.task_id = task_id
greenlet.link_exception(self.handle_killed_greenlets)
self.greenlets.append(greenlet)
return task_id
def query_task_result(self, task_id):
logger.debug("Querying task result with task id {}".format(task_id))
with self.task_lock:
len1 = len(self.task_results)
ret = self.task_results.pop(int(task_id), None)
if not ret and len1 != len(self.task_results):
logger.error("Popped None from results task but lost an entry")
if ret:
logger.debug("Found response for task {}".format(task_id))
return ret
def get_fiat_exchange_rates(self, currencies):
res = {'exchange_rates': get_fiat_usd_exchange_rates(currencies)}
return process_result(res)
def get_settings(self):
return process_result(self.rotkehlchen.data.db.get_settings())
def remove_exchange(self, name):
result, message = self.rotkehlchen.remove_exchange(name)
return {'result': result, 'message': message}
def setup_exchange(self, name, api_key, api_secret):
result, message = self.rotkehlchen.setup_exchange(
name, api_key, api_secret)
return {'result': result, 'message': message}
def query_otctrades(self):
trades = self.rotkehlchen.data.get_external_trades()
return process_result(trades)
def add_otctrade(self, data):
result, message = self.rotkehlchen.data.add_external_trade(data)
return {'result': result, 'message': message}
def edit_otctrade(self, data):
result, message = self.rotkehlchen.data.edit_external_trade(data)
return {'result': result, 'message': message}
def delete_otctrade(self, trade_id):
result, message = self.rotkehlchen.data.delete_external_trade(trade_id)
return {'result': result, 'message': message}
def set_premium_credentials(self, api_key, api_secret):
result, empty_or_error = self.rotkehlchen.set_premium_credentials(
api_key, api_secret)
return {'result': result, 'message': empty_or_error}
def set_premium_option_sync(self, should_sync):
self.rotkehlchen.data.db.update_premium_sync(should_sync)
return True
def query_exchange_balances(self, name):
balances = getattr(self.rotkehlchen, name).query_balances()
res = {'name': name, 'balances': balances}
return process_result(res)
def query_exchange_balances_async(self, name):
res = self.query_async('query_exchange_balances', name=name)
return {'task_id': res}
def query_blockchain_balances(self):
balances = self.rotkehlchen.blockchain.query_balances()
return process_result(balances)
def query_blockchain_balances_async(self):
res = self.query_async('query_blockchain_balances')
return {'task_id': res}
def query_fiat_balances(self):
res = self.rotkehlchen.query_fiat_balances()
return process_result(res)
def set_fiat_balance(self, currency, balance):
result, message = self.rotkehlchen.data.set_fiat_balance(
currency, balance)
return {'result': result, 'message': message}
def query_trade_history(self, location, start_ts, end_ts):
start_ts = int(start_ts)
end_ts = int(end_ts)
if location == 'all':
return self.rotkehlchen.trades_historian.get_history(
start_ts, end_ts)
try:
exchange = getattr(self.rotkehlchen, location)
except AttributeError:
raise "Unknown location {} given".format(location)
return exchange.query_trade_history(start_ts, end_ts)
def query_asset_price(self, from_asset, to_asset, timestamp):
price = self.rotkehlchen.data.accountant.query_historical_price(
from_asset, to_asset, int(timestamp))
return str(price)
def process_trade_history(self, start_ts, end_ts):
start_ts = int(start_ts)
end_ts = int(end_ts)
result = self.rotkehlchen.process_history(start_ts, end_ts)
return process_result(result)
def process_trade_history_async(self, start_ts, end_ts):
res = self.query_async('process_trade_history',
start_ts=start_ts,
end_ts=end_ts)
return {'task_id': res}
def query_balances(self, save_data=False):
if isinstance(save_data, str) and (save_data == 'save'
or save_data == 'True'):
save_data = True
result = self.rotkehlchen.query_balances(save_data)
print(pretty_json_dumps(result))
return process_result(result)
def query_balances_async(self, save_data=False):
res = self.query_async('query_balances')
return {'task_id': res}
def get_eth_tokens(self):
result = {
'all_eth_tokens': self.rotkehlchen.data.eth_tokens,
'owned_eth_tokens': self.rotkehlchen.blockchain.eth_tokens
}
return process_result(result)
def add_owned_eth_tokens(self, tokens):
try:
new_data = self.rotkehlchen.blockchain.track_new_tokens(tokens)
except InputError as e:
return simple_result(False, str(e))
self.rotkehlchen.data.write_owned_eth_tokens(
self.rotkehlchen.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_owned_eth_tokens(self, tokens):
try:
new_data = self.rotkehlchen.blockchain.remove_eth_tokens(tokens)
except InputError as e:
return simple_result(False, str(e))
self.rotkehlchen.data.write_owned_eth_tokens(
self.rotkehlchen.blockchain.owned_eth_tokens)
return accounts_result(new_data['per_account'], new_data['totals'])
def add_blockchain_account(self, blockchain, account):
try:
new_data = self.rotkehlchen.blockchain.add_blockchain_account(
blockchain, account)
except InputError as e:
return simple_result(False, str(e))
self.rotkehlchen.data.add_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def remove_blockchain_account(self, blockchain, account):
try:
new_data = self.rotkehlchen.blockchain.remove_blockchain_account(
blockchain, account)
except InputError as e:
return simple_result(False, str(e))
self.rotkehlchen.data.remove_blockchain_account(blockchain, account)
return accounts_result(new_data['per_account'], new_data['totals'])
def get_ignored_assets(self):
result = {
'ignored_assets': self.rotkehlchen.data.db.get_ignored_assets()
}
return result
def add_ignored_asset(self, asset):
result, message = self.rotkehlchen.data.add_ignored_asset(asset)
return {'result': result, 'message': message}
def remove_ignored_asset(self, asset):
result, message = self.rotkehlchen.data.remove_ignored_asset(asset)
return {'result': result, 'message': message}
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
"""Either unlock an existing user or create a new one"""
res = {'result': True, 'message': ''}
assert isinstance(sync_approval,
str), "sync_approval should be a string"
assert isinstance(api_key, str), "api_key should be a string"
assert isinstance(api_secret, str), "api_secret should be a string"
valid_approve = isinstance(
sync_approval, str) and sync_approval in ['unknown', 'yes', 'no']
if not valid_approve:
raise ValueError('Provided invalid value for sync_approval')
if api_key != '' and create_new is False:
raise ValueError(
'Should not ever have api_key provided during a normal login')
if api_key != '' and api_secret == '' or api_secret != '' and api_key == '':
raise ValueError('Must provide both or neither of api key/secret')
try:
self.rotkehlchen.unlock_user(user, password, create_new,
sync_approval, api_key, api_secret)
res['exchanges'] = self.rotkehlchen.connected_exchanges
res['premium'] = True if hasattr(self.rotkehlchen,
'premium') else False
res['settings'] = self.rotkehlchen.data.db.get_settings()
except AuthenticationError as e:
res['result'] = False
res['message'] = str(e)
except PermissionError as e:
res['result'] = False
res['permission_needed'] = True
res['message'] = str(e)
return res
def echo(self, text):
return text
def main(self):
gevent.hub.signal(signal.SIGINT, self.shutdown)
gevent.hub.signal(signal.SIGTERM, self.shutdown)
# self.zerorpc = zerorpc.Server(self, heartbeat=15)
self.zerorpc = zerorpc.Server(self)
addr = 'tcp://127.0.0.1:' + str(self.port())
self.zerorpc.bind(addr)
print('start running on {}'.format(addr))
self.zerorpc.run()
def test_initializing_rotki_with_datadir_with_wrong_permissions(
cli_args, data_dir):
os.chmod(data_dir, 0o200)
with pytest.raises(SystemPermissionError):
Rotkehlchen(args=cli_args)
os.chmod(data_dir, 0o777)
def test_periodic_data_before_login_completion(cli_args):
"""Test that periodic query returns empty list if user is not yet logged in"""
rotkehlchen = Rotkehlchen(cli_args)
result = rotkehlchen.query_periodic_data()
assert len(result) == 0
class RotkehlchenServer(object):
def __init__(self):
self.args = app_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
# Greenlets that will be waited for when we shutdown
self.waited_greenlets = [mainloop_greenlet]
# Greenlets that can be killed instead of waited for when we shutdown
self.killable_greenlets = []
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def new_task_id(self):
with self.task_lock:
task_id = self.task_id
self.task_id += 1
return task_id
def write_task_result(self, task_id, result):
with self.task_lock:
self.task_results[task_id] = result
def get_task_result(self, task_id):
with self.task_lock:
return self.task_results[task_id]
def port(self):
return self.args.zerorpc_port
def shutdown(self):
log.debug('Shutdown initiated')
self.zerorpc.stop()
self.rotkehlchen.shutdown()
log.debug('Waiting for greenlets')
gevent.wait(self.waited_greenlets)
log.debug('Waited for greenlets. Killing all other greenlets')
gevent.killall(self.killable_greenlets)
log.debug('Greenlets killed. Killing zerorpc greenlet')
self.zerorpc_greenlet.kill()
log.debug('Killed zerorpc greenlet')
log.debug('Shutdown completed')
logging.shutdown()
self.stop_event.set()
def logout(self):
# Kill all queries apart from the main loop -- perhaps a bit heavy handed
# but the other options would be:
# 1. to wait for all of them. That could take a lot of time, for no reason.
# All results would be discarded anyway since we are logging out.
# 2. Have an intricate stop() notification system for each greenlet, but
# that is going to get complicated fast.
gevent.killall(self.killable_greenlets)
with self.task_lock:
self.task_results = {}
self.rotkehlchen.logout()
def set_main_currency(self, currency_text):
self.rotkehlchen.set_main_currency(currency_text)
def set_settings(self, settings):
result, message = self.rotkehlchen.set_settings(settings)
return {'result': result, 'message': message}
def get_total_in_usd(self, balances):
total = 0
for _, entry in balances.items():
total += entry['usd_value']
return total
def handle_killed_greenlets(self, greenlet):
if not greenlet.exception:
log.warning('handle_killed_greenlets without an exception')
return
log.error(
'Greenlet for task {} dies with exception: {}.\n'
'Exception Name: {}\nException Info: {}\nTraceback:\n {}'.format(
greenlet.task_id,
greenlet.exception,
greenlet.exc_info[0],
greenlet.exc_info[1],
''.join(traceback.format_tb(greenlet.exc_info[2])),
))
# also write an error for the task result
result = {
'error': str(greenlet.exception),
}
self.write_task_result(greenlet.task_id, result)
def _query_async(self, command, task_id, **kwargs):
result = getattr(self, command)(**kwargs)
self.write_task_result(task_id, result)
def query_async(self, command, **kwargs):
task_id = self.new_task_id()
log.debug("NEW TASK {} (kwargs:{}) with ID: {}".format(
command, kwargs, task_id))
greenlet = gevent.spawn(
self._query_async,
command,
task_id,
**kwargs,
)
greenlet.task_id = task_id
greenlet.link_exception(self.handle_killed_greenlets)
self.killable_greenlets.append(greenlet)
return task_id
def query_task_result(self, task_id):
with self.task_lock:
len1 = len(self.task_results)
ret = self.task_results.pop(int(task_id), None)
if not ret and len1 != len(self.task_results):
log.error("Popped None from results task but lost an entry")
if ret:
log.debug("Found response for task {}".format(task_id))
return ret
def get_fiat_exchange_rates(self, currencies):
rates = self.rotkehlchen.inquirer.get_fiat_usd_exchange_rates(
currencies)
res = {'exchange_rates': rates}
return process_result(res)
def get_settings(self):
return process_result(self.rotkehlchen.data.db.get_settings())
def remove_exchange(self, name):
result, message = self.rotkehlchen.remove_exchange(name)
return {'result': result, 'message': message}
def setup_exchange(self, name, api_key, api_secret):
result, message = self.rotkehlchen.setup_exchange(
name, api_key, api_secret)
return {'result': result, 'message': message}
def query_otctrades(self):
trades = self.rotkehlchen.data.get_external_trades()
return process_result(trades)
def add_otctrade(self, data):
result, message = self.rotkehlchen.data.add_external_trade(data)
return {'result': result, 'message': message}
def edit_otctrade(self, data):
result, message = self.rotkehlchen.data.edit_external_trade(data)
return {'result': result, 'message': message}
def delete_otctrade(self, trade_id):
result, message = self.rotkehlchen.data.delete_external_trade(trade_id)
return {'result': result, 'message': message}
def set_premium_credentials(self, api_key, api_secret):
result, empty_or_error = self.rotkehlchen.set_premium_credentials(
api_key, api_secret)
return {'result': result, 'message': empty_or_error}
def set_premium_option_sync(self, should_sync):
self.rotkehlchen.data.db.update_premium_sync(should_sync)
return True
def query_exchange_balances(self, name):
res = {'name': name}
balances, msg = getattr(self.rotkehlchen, name).query_balances()
if balances is None:
res['error'] = msg
else:
res['balances'] = balances
return process_result(res)
def query_exchange_balances_async(self, name):
res = self.query_async('query_exchange_balances', name=name)
return {'task_id': res}
def query_blockchain_balances(self):
result, empty_or_error = self.rotkehlchen.blockchain.query_balances()
return process_result({'result': result, 'message': empty_or_error})
def query_blockchain_balances_async(self):
res = self.query_async('query_blockchain_balances')
return {'task_id': res}
def query_fiat_balances(self):
res = self.rotkehlchen.query_fiat_balances()
return process_result(res)
def set_fiat_balance(self, currency, balance):
result, message = self.rotkehlchen.data.set_fiat_balance(
currency, balance)
return {'result': result, 'message': message}
def query_trade_history(self, location, start_ts, end_ts):
start_ts = int(start_ts)
end_ts = int(end_ts)
if location == 'all':
return self.rotkehlchen.trades_historian.get_history(
start_ts, end_ts)
try:
exchange = getattr(self.rotkehlchen, location)
except AttributeError:
raise "Unknown location {} given".format(location)
return process_result(
exchange.query_trade_history(start_ts, end_ts, end_ts))
def process_trade_history(self, start_ts, end_ts):
start_ts = int(start_ts)
end_ts = int(end_ts)
result, error_or_empty = self.rotkehlchen.process_history(
start_ts, end_ts)
response = {'result': result, 'message': error_or_empty}
return process_result(response)
def process_trade_history_async(self, start_ts, end_ts):
res = self.query_async('process_trade_history',
start_ts=start_ts,
end_ts=end_ts)
return {'task_id': res}
def export_processed_history_csv(self, dirpath):
result, message = self.rotkehlchen.accountant.csvexporter.create_files(
dirpath)
return {'result': result, 'message': message}
def query_balances(self, save_data=False):
if isinstance(save_data, str) and (save_data == 'save'
or save_data == 'True'):
save_data = True
result = self.rotkehlchen.query_balances(save_data)
print(pretty_json_dumps(result))
return process_result(result)
def query_balances_async(self, save_data=False):
res = self.query_async('query_balances')
return {'task_id': res}
def query_periodic_data(self):
"""Will query for some client data that can change frequently"""
result = self.rotkehlchen.query_periodic_data()
return process_result(result)
def get_eth_tokens(self):
result = {
'all_eth_tokens': self.rotkehlchen.data.eth_tokens,
'owned_eth_tokens': self.rotkehlchen.blockchain.eth_tokens,
}
return process_result(result)
def add_owned_eth_tokens(self, tokens):
return self.rotkehlchen.add_owned_eth_tokens(tokens)
def remove_owned_eth_tokens(self, tokens):
return self.rotkehlchen.remove_owned_eth_tokens(tokens)
def add_blockchain_account(self, blockchain, account):
return self.rotkehlchen.add_blockchain_account(blockchain, account)
def remove_blockchain_account(self, blockchain, account):
return self.rotkehlchen.remove_blockchain_account(blockchain, account)
def get_ignored_assets(self):
result = {
'ignored_assets': self.rotkehlchen.data.db.get_ignored_assets(),
}
return result
def add_ignored_asset(self, asset):
result, message = self.rotkehlchen.data.add_ignored_asset(asset)
return {'result': result, 'message': message}
def remove_ignored_asset(self, asset):
result, message = self.rotkehlchen.data.remove_ignored_asset(asset)
return {'result': result, 'message': message}
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
"""Either unlock an existing user or create a new one"""
res = {'result': True, 'message': ''}
assert isinstance(sync_approval,
str), "sync_approval should be a string"
assert isinstance(api_key, str), "api_key should be a string"
assert isinstance(api_secret, str), "api_secret should be a string"
valid_actions = ['unknown', 'yes', 'no']
valid_approve = isinstance(sync_approval,
str) and sync_approval in valid_actions
if not valid_approve:
raise ValueError('Provided invalid value for sync_approval')
if api_key != '' and create_new is False:
raise ValueError(
'Should not ever have api_key provided during a normal login')
if api_key != '' and api_secret == '' or api_secret != '' and api_key == '':
raise ValueError('Must provide both or neither of api key/secret')
try:
self.rotkehlchen.unlock_user(
user,
password,
create_new,
sync_approval,
api_key,
api_secret,
)
res['exchanges'] = self.rotkehlchen.connected_exchanges
res['premium'] = self.rotkehlchen.premium is not None
res['settings'] = self.rotkehlchen.data.db.get_settings()
except AuthenticationError as e:
res['result'] = False
res['message'] = str(e)
except RotkehlchenPermissionError as e:
res['result'] = False
res['permission_needed'] = True
res['message'] = str(e)
return res
def echo(self, text):
return text
def main(self):
gevent.hub.signal(signal.SIGQUIT, self.shutdown)
gevent.hub.signal(signal.SIGINT, self.shutdown)
gevent.hub.signal(signal.SIGTERM, self.shutdown)
# self.zerorpc = zerorpc.Server(self, heartbeat=15)
self.zerorpc = zerorpc.Server(self)
addr = 'tcp://127.0.0.1:' + str(self.port())
self.zerorpc.bind(addr)
print('start running on {}'.format(addr))
self.zerorpc_greenlet = gevent.spawn(self.zerorpc.run)
self.stop_event.wait()
def uninitialized_rotkehlchen(cli_args):
"""A rotkehlchen instance that has only had __init__ run but is not unlocked"""
return Rotkehlchen(cli_args)
class RotkehlchenServer():
def __init__(self):
arg_parser = app_args(
prog='rotkehlchen',
description='Rotkehlchen Crypto Portfolio Management',
)
self.args = arg_parser.parse_args()
self.rotkehlchen = Rotkehlchen(self.args)
self.stop_event = Event()
mainloop_greenlet = self.rotkehlchen.start()
mainloop_greenlet.link_exception(self.handle_killed_greenlets)
# Greenlets that will be waited for when we shutdown
self.waited_greenlets = [mainloop_greenlet]
# Greenlets that can be killed instead of waited for when we shutdown
self.killable_greenlets = []
self.task_lock = Semaphore()
self.task_id = 0
self.task_results = {}
def new_task_id(self):
with self.task_lock:
task_id = self.task_id
self.task_id += 1
return task_id
def write_task_result(self, task_id, result):
with self.task_lock:
self.task_results[task_id] = result
def get_task_result(self, task_id):
with self.task_lock:
return self.task_results[task_id]
def port(self):
return self.args.zerorpc_port
def shutdown(self):
log.debug('Shutdown initiated')
self.zerorpc.stop()
self.rotkehlchen.shutdown()
log.debug('Waiting for greenlets')
gevent.wait(self.waited_greenlets)
log.debug('Waited for greenlets. Killing all other greenlets')
gevent.killall(self.killable_greenlets)
log.debug('Greenlets killed. Killing zerorpc greenlet')
self.zerorpc_greenlet.kill()
log.debug('Killed zerorpc greenlet')
log.debug('Shutdown completed')
logging.shutdown()
self.stop_event.set()
def logout(self):
# Kill all queries apart from the main loop -- perhaps a bit heavy handed
# but the other options would be:
# 1. to wait for all of them. That could take a lot of time, for no reason.
# All results would be discarded anyway since we are logging out.
# 2. Have an intricate stop() notification system for each greenlet, but
# that is going to get complicated fast.
gevent.killall(self.killable_greenlets)
with self.task_lock:
self.task_results = {}
self.rotkehlchen.logout()
def set_main_currency(self, currency_text):
self.rotkehlchen.set_main_currency(currency_text)
def set_settings(self, settings):
result, message = self.rotkehlchen.set_settings(settings)
return {'result': result, 'message': message}
def handle_killed_greenlets(self, greenlet):
if not greenlet.exception:
log.warning('handle_killed_greenlets without an exception')
return
log.error(
'Greenlet for task {} dies with exception: {}.\n'
'Exception Name: {}\nException Info: {}\nTraceback:\n {}'.format(
greenlet.task_id,
greenlet.exception,
greenlet.exc_info[0],
greenlet.exc_info[1],
''.join(traceback.format_tb(greenlet.exc_info[2])),
))
# also write an error for the task result
result = {
'error': str(greenlet.exception),
}
self.write_task_result(greenlet.task_id, result)
def _query_async(self, command, task_id, **kwargs):
result = getattr(self, command)(**kwargs)
self.write_task_result(task_id, result)
def query_async(self, command, **kwargs):
task_id = self.new_task_id()
log.debug("NEW TASK {} (kwargs:{}) with ID: {}".format(
command, kwargs, task_id))
greenlet = gevent.spawn(
self._query_async,
command,
task_id,
**kwargs,
)
greenlet.task_id = task_id
greenlet.link_exception(self.handle_killed_greenlets)
self.killable_greenlets.append(greenlet)
return task_id
def query_task_result(self, task_id):
with self.task_lock:
len1 = len(self.task_results)
ret = self.task_results.pop(int(task_id), None)
if not ret and len1 != len(self.task_results):
log.error("Popped None from results task but lost an entry")
if ret:
log.debug("Found response for task {}".format(task_id))
return ret
@staticmethod
def get_fiat_exchange_rates(currencies: List[str]):
fiat_currencies = cast(List[FiatAsset], currencies)
rates = Inquirer().get_fiat_usd_exchange_rates(fiat_currencies)
res = {'exchange_rates': rates}
return process_result(res)
def get_settings(self):
return process_result(self.rotkehlchen.data.db.get_settings())
def remove_exchange(self, name):
result, message = self.rotkehlchen.remove_exchange(name)
return {'result': result, 'message': message}
def setup_exchange(self, name, api_key, api_secret):
result, message = self.rotkehlchen.setup_exchange(
name, api_key, api_secret)
return {'result': result, 'message': message}
def query_otctrades(self):
trades = self.rotkehlchen.data.get_external_trades()
result = {'result': trades, 'message': ''}
return process_result(result)
def add_otctrade(self, data):
result, message = self.rotkehlchen.data.add_external_trade(data)
return {'result': result, 'message': message}
def edit_otctrade(self, data):
result, message = self.rotkehlchen.data.edit_external_trade(data)
return {'result': result, 'message': message}
def delete_otctrade(self, trade_id):
result, message = self.rotkehlchen.data.delete_external_trade(trade_id)
return {'result': result, 'message': message}
def set_premium_credentials(self, api_key, api_secret):
msg = ''
result = False
try:
self.rotkehlchen.set_premium_credentials(api_key, api_secret)
result = True
except (AuthenticationError, IncorrectApiKeyFormat) as e:
msg = str(e)
return {'result': result, 'message': msg}
def set_premium_option_sync(self, should_sync):
self.rotkehlchen.data.db.update_premium_sync(should_sync)
return True
def query_exchange_balances(self, name):
res = {'name': name}
exchange_obj = self.rotkehlchen.exchange_manager.connected_exchanges.get(
name, None)
if not exchange_obj:
res['error'] = f'Could not query balances for {name} since it is not registered'
return process_result(res)
balances, msg = exchange_obj.query_balances()
if balances is None:
res['error'] = msg
else:
res['balances'] = balances
return process_result(res)
def query_exchange_balances_async(self, name):
res = self.query_async('query_exchange_balances', name=name)
return {'task_id': res}
def query_blockchain_balances(self):
result, empty_or_error = self.rotkehlchen.blockchain.query_balances()
return process_result({'result': result, 'message': empty_or_error})
def query_blockchain_balances_async(self):
res = self.query_async('query_blockchain_balances')
return {'task_id': res}
def query_fiat_balances(self):
res = self.rotkehlchen.query_fiat_balances()
return process_result(res)
def query_netvalue_data(self):
res = self.rotkehlchen.data.db.get_netvalue_data()
result = {'times': res[0], 'data': res[1]}
return process_result(result)
def query_timed_balances_data(self, given_asset: str, start_ts: int,
end_ts: int) -> Dict:
try:
start_ts = deserialize_timestamp(start_ts)
end_ts = deserialize_timestamp(end_ts)
asset = Asset(given_asset)
except (UnknownAsset, DeserializationError) as e:
return {'result': False, 'message': str(e)}
res = self.rotkehlchen.data.db.query_timed_balances(
from_ts=start_ts,
to_ts=end_ts,
asset=asset,
)
result = {'result': res, 'message': ''}
return process_result(result)
def query_owned_assets(self):
res = self.rotkehlchen.data.db.query_owned_assets()
result = {'result': res, 'message': ''}
return process_result(result)
def query_latest_location_value_distribution(self):
res = self.rotkehlchen.data.db.get_latest_location_value_distribution()
result = {'result': res, 'message': ''}
return process_result(result)
def query_latest_asset_value_distribution(self):
res = self.rotkehlchen.data.db.get_latest_asset_value_distribution()
result = {'result': res, 'message': ''}
return process_result(result)
def consume_messages(self):
"""Consumes all errors and warnings from the messages aggregator"""
warnings = self.rotkehlchen.msg_aggregator.consume_warnings()
errors = self.rotkehlchen.msg_aggregator.consume_errors()
result = {
'result': {
'warnings': warnings,
'errors': errors
},
'message': ''
}
return process_result(result)
def query_statistics_renderer(self):
result_dict = {'result': '', 'message': 'user does not have premium'}
if not self.rotkehlchen.premium:
return process_result(result_dict)
active = self.rotkehlchen.premium.is_active()
if not active:
return process_result(result_dict)
try:
result = self.rotkehlchen.premium.query_statistics_renderer()
result_dict['result'] = result
result_dict['message'] = ''
except RemoteError as e:
result_dict['message'] = str(e)
return process_result(result_dict)
def set_fiat_balance(self, currency: str, balance: str):
result, message = self.rotkehlchen.data.set_fiat_balance(
currency, balance)
return {'result': result, 'message': message}
def query_trade_history(self, location: str, start_ts: int, end_ts: int):
"""Queries the trades/margin position history of a single or all exchanges
Note: This will only query trades/margin position history. Nothing else.
Not loans, deposit/withdrawals e.t.c.
"""
start_ts = Timestamp(start_ts)
end_ts = Timestamp(end_ts)
if location == 'all':
(
empty_or_error,
history,
_,
_,
_,
) = self.rotkehlchen.trades_historian.get_history(
start_ts, end_ts)
if empty_or_error != '':
return process_result({
'result': '',
'message': empty_or_error
})
# Ignore everything except for trades
return process_result({'result': history, 'message': ''})
if location not in SUPPORTED_EXCHANGES:
return {
'result':
'',
'message':
f'Unknown exchange {location} provided in query_trade_history',
}
exchange = self.rotkehlchen.exchange_manager.connected_exchanges.get(
location, None)
if not exchange:
msg = (
f'Exchange {location} provided in query_trade_history is not '
f'registered yet for this user')
return {'result': '', 'message': msg}
result = exchange.query_trade_history(start_ts, end_ts, end_ts)
return process_result({'result': result, 'message': ''})
def process_trade_history(self, start_ts, end_ts):
start_ts = int(start_ts)
end_ts = int(end_ts)
result, error_or_empty = self.rotkehlchen.process_history(
start_ts, end_ts)
response = {'result': result, 'message': error_or_empty}
return process_result(response)
def process_trade_history_async(self, start_ts, end_ts):
res = self.query_async('process_trade_history',
start_ts=start_ts,
end_ts=end_ts)
return {'task_id': res}
def export_processed_history_csv(self, dirpath):
result, message = self.rotkehlchen.accountant.csvexporter.create_files(
dirpath)
return {'result': result, 'message': message}
def query_balances(self, save_data=False):
if isinstance(save_data, str) and (save_data == 'save'
or save_data == 'True'):
save_data = True
result = self.rotkehlchen.query_balances(save_data)
print(pretty_json_dumps(result))
return process_result(result)
def query_balances_async(self, save_data=False):
res = self.query_async('query_balances', save_data=save_data)
return {'task_id': res}
def query_periodic_data(self):
"""Will query for some client data that can change frequently"""
result = self.rotkehlchen.query_periodic_data()
return process_result(result)
def get_eth_tokens(self):
result = {
'all_eth_tokens': self.rotkehlchen.data.eth_tokens,
'owned_eth_tokens': self.rotkehlchen.blockchain.eth_tokens,
}
return process_result(result)
def add_owned_eth_tokens(self, tokens):
return self.rotkehlchen.add_owned_eth_tokens(tokens)
def remove_owned_eth_tokens(self, tokens):
return self.rotkehlchen.remove_owned_eth_tokens(tokens)
def add_blockchain_account(self, given_blockchain: str,
given_account: str):
try:
blockchain = SupportedBlockchain(given_blockchain)
except ValueError:
msg = f'Tried to add blockchain account for unsupported blockchain {given_blockchain}'
return simple_result(False, msg)
return self.rotkehlchen.add_blockchain_account(blockchain,
given_account)
def remove_blockchain_account(self, given_blockchain: str,
given_account: str):
try:
blockchain = SupportedBlockchain(given_blockchain)
except ValueError:
msg = (
f'Tried to remove blockchain account for unsupported blockchain {given_blockchain}'
)
return simple_result(False, msg)
return self.rotkehlchen.remove_blockchain_account(
blockchain, given_account)
def get_ignored_assets(self):
result = {
'ignored_assets': [
identifier for identifier in
self.rotkehlchen.data.db.get_ignored_assets()
],
}
return process_result(result)
def add_ignored_asset(self, asset: str):
result, message = self.rotkehlchen.data.add_ignored_asset(asset)
return {'result': result, 'message': message}
def remove_ignored_asset(self, asset: str):
result, message = self.rotkehlchen.data.remove_ignored_asset(asset)
return {'result': result, 'message': message}
@staticmethod
def version_check():
result = check_if_version_up_to_date()
if not result:
our_version = None
latest_version = None
url = None
else:
our_version = result[0]
latest_version = result[1]
url = result[2]
return {
'our_version': our_version,
'latest_version': latest_version,
'url': url
}
def unlock_user(
self,
user: str,
password: str,
create_new: Union[bool, str],
sync_approval: str,
api_key: str,
api_secret: str,
) -> Dict[str, Any]:
"""Either unlock an existing user or create a new one"""
res = {'result': True, 'message': ''}
assert isinstance(sync_approval,
str), "sync_approval should be a string"
assert isinstance(api_key, str), "api_key should be a string"
assert isinstance(api_secret, str), "api_secret should be a string"
if not isinstance(create_new, bool):
if not isinstance(create_new, str):
raise ValueError('create_new can only be boolean or str')
if create_new in ('False', 'false', 'FALSE'):
create_new = False
elif create_new in ('True', 'true', 'TRUE'):
create_new = True
else:
raise ValueError(
f'Invalid string value for create_new {create_new}')
valid_actions = ['unknown', 'yes', 'no']
if sync_approval not in valid_actions:
raise ValueError('Provided invalid value for sync_approval')
if api_key != '' and create_new is False:
raise ValueError(
'Should not ever have api_key provided during a normal login')
if api_key != '' and api_secret == '' or api_secret != '' and api_key == '':
raise ValueError('Must provide both or neither of api key/secret')
try:
self.rotkehlchen.unlock_user(
user,
password,
create_new,
sync_approval,
api_key,
api_secret,
)
res['exchanges'] = self.rotkehlchen.exchange_manager.get_connected_exchange_names(
)
res['premium'] = self.rotkehlchen.premium is not None
res['settings'] = self.rotkehlchen.data.db.get_settings()
except AuthenticationError as e:
res['result'] = False
res['message'] = str(e)
except RotkehlchenPermissionError as e:
res['result'] = False
res['permission_needed'] = True
res['message'] = str(e)
return res
def main(self):
if os.name != 'nt':
gevent.hub.signal(signal.SIGQUIT, self.shutdown)
gevent.hub.signal(signal.SIGINT, self.shutdown)
gevent.hub.signal(signal.SIGTERM, self.shutdown)
self.zerorpc = zerorpc.Server(self)
addr = 'tcp://127.0.0.1:' + str(self.port())
self.zerorpc.bind(addr)
print('start running on {}'.format(addr))
self.zerorpc_greenlet = gevent.spawn(self.zerorpc.run)
self.stop_event.wait()
def rotkehlchen_instance(cli_args, username, blockchain):
r = Rotkehlchen(cli_args)
r.data.unlock(username, '123', create_new=True)
r.blockchain = blockchain
return r
