def fixture_data_dir(use_clean_caching_directory, tmpdir_factory) -> Path:
"""The tests data dir is peristent so that we can cache price queries between
tests. If use_clean_caching_directory is True then a completely fresh dir is returned"""
if use_clean_caching_directory:
return Path(tmpdir_factory.mktemp('test_data_dir'))
if 'CI' in os.environ:
data_directory = Path.home() / '.cache' / '.rotkehlchen-test-dir'
else:
data_directory = default_data_directory().parent / 'test_data'
data_directory.mkdir(parents=True, exist_ok=True)
# do not keep pull github assets between tests. Can really confuse test results
# as we may end up with different set of assets in tests
(data_directory / 'assets').unlink(missing_ok=True)
# Remove any old accounts. The only reason we keep this directory around is for
# cached price queries, not for user DBs
for x in data_directory.iterdir():
directory_with_db = (x.is_dir() and (x / 'rotkehlchen.db').exists()
or (x / 'rotkehlchen_transient.db').exists())
if directory_with_db:
shutil.rmtree(x, ignore_errors=True)
return data_directory
def __init__(self, args: argparse.Namespace) -> None:
self.db = DBHandler(
user_data_dir=default_data_directory() / args.user_name,
password=args.user_password,
msg_aggregator=MessagesAggregator(),
initial_settings=None,
)
def app_args(prog: str, description: str) -> argparse.ArgumentParser:
"""Add the Rotkehlchen arguments to the argument parser and return it"""
p = argparse.ArgumentParser(
prog=prog,
description=description,
)
p.add_argument(
'--sleep-secs',
type=int,
default=20,
help="Seconds to sleep during the main loop",
)
p.add_argument(
'--data-dir',
help='The directory where all data and configs are placed',
default=default_data_directory(),
)
p.add_argument(
'--zerorpc-port',
help=
'The port on which to open a zerorpc server for communication with the UI',
default=4242,
)
p.add_argument(
'--ethrpc-port',
help="The port on which to communicate with an ethereum client's RPC.",
default=8545,
)
p.add_argument(
'--logfile',
help='The name of the file to write log entries to',
default='rotkehlchen.log',
)
p.add_argument(
'--logtarget',
help=
'Choose where logging entries will be sent. Valid values are "file and "stdout"',
choices=['stdout', 'file'],
default='file',
)
p.add_argument(
'--loglevel',
help='Choose the logging level',
choices=['debug', 'info', 'warn', 'error', 'critical'],
default='debug',
)
p.add_argument(
'--logfromothermodules',
help=('If given then logs from all imported modules that use the '
'logging system will also be visible.'),
action='store_true',
)
p.add_argument(
'version',
help='Shows the rotkehlchen version',
action=VersionAction,
)
return p
def __init__(self, args: argparse.Namespace) -> None:
user_path = FilePath(
os.path.join(str(default_data_directory()), args.user_name))
self.db = DBHandler(
user_data_dir=user_path,
password=args.user_password,
msg_aggregator=MessagesAggregator(),
)
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator)
# Initialize the AssetResolver singleton
AssetResolver(data_directory=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko)
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='periodically_query_icons_until_all_cached',
method=self.icon_manager.periodically_query_icons_until_all_cached,
batch_size=ICONS_BATCH_SIZE,
sleep_time_secs=ICONS_QUERY_SLEEP,
)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.lock.release()
self.shutdown_event = gevent.event.Event()
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
This runs during backend initialization so it should be as light as possible.
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
self.data_dir.mkdir(parents=True, exist_ok=True)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.main_loop_spawned = False
self.args = args
self.api_task_greenlets: List[gevent.Greenlet] = []
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
# Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver
GlobalDBHandler(data_dir=self.data_dir)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
self.coingecko = Coingecko()
self.icon_manager = IconManager(data_dir=self.data_dir,
coingecko=self.coingecko)
self.assets_updater = AssetsUpdater(self.msg_aggregator)
# Initialize the Inquirer singleton
Inquirer(
data_dir=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
# Keeps how many trades we have found per location. Used for free user limiting
self.actions_per_location: Dict[str, Dict[Location, int]] = {
'trade': defaultdict(int),
'asset_movement': defaultdict(int),
}
self.task_manager: Optional[TaskManager] = None
self.shutdown_event = gevent.event.Event()
def app_args() -> argparse.Namespace:
""" Parse the arguments and create and return the arguments object"""
p = argparse.ArgumentParser(
description='Rotkehlchen Crypto Portfolio Management')
p.add_argument(
'--output',
help=('A path to a file for logging all output. If nothing is given'
'stdout is used'))
p.add_argument('--sleep-secs',
type=int,
default=20,
help="Seconds to sleep during the main loop")
p.add_argument('--notify',
action='store_true',
help=('If given then the tool will send notifications via '
'notify-send.'))
p.add_argument('--data-dir',
help='The directory where all data and configs are placed',
default=default_data_directory())
p.add_argument(
'--zerorpc-port',
help=
'The port on which to open a zerorpc server for communication with the UI',
default=4242)
p.add_argument(
'--ethrpc-port',
help="The port on which to communicate with an ethereum client's RPC.",
default=8545,
)
p.add_argument(
'--logfile',
help='The name of the file to write log entries to',
default='rotkehlchen.log',
)
p.add_argument(
'--logtarget',
help=
'Choose where logging entries will be sent. Valid values are "file and "stdout"',
choices=['stdout', 'file'],
default='file',
)
p.add_argument('--loglevel',
help='Choose the logging level',
choices=['debug', 'info', 'warn', 'error', 'critical'],
default='debug')
p.add_argument(
'--logfromothermodules',
help=('If given then logs from all imported modules that use the '
'logging system will also be visible.'),
action='store_true',
)
args = p.parse_args()
return args
def data_dir(use_clean_caching_directory, tmpdir_factory) -> Path:
"""The tests data dir is peristent so that we can cache price queries between
tests. If use_clean_caching_directory is True then a completely fresh dir is returned"""
if use_clean_caching_directory:
return Path(tmpdir_factory.mktemp('test_data_dir'))
if 'CI' in os.environ:
data_directory = Path.home() / '.cache' / '.rotkehlchen-test-dir'
else:
data_directory = default_data_directory().parent / 'test_data'
data_directory.mkdir(parents=True, exist_ok=True)
# Remove any old accounts. The only reason we keep this directory around is for
# cached price queries, not for user DBs
for x in data_directory.iterdir():
if x.is_dir() and (x / 'rotkehlchen.db').exists():
shutil.rmtree(x)
return data_directory
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in: bool = False
configure_logging(args)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare)
self.lock.release()
self.shutdown_event = gevent.event.Event()
def find_coingecko_by_id(identifier: str, coins):
for coingecko_entry in coins:
if coingecko_entry['id'] == identifier:
return coingecko_entry
return None
root_dir = Path(__file__).resolve().parent.parent.parent
ASSETS_FILE = Path(f'{root_dir}/rotkehlchen/data/all_assets.json')
with open(ASSETS_FILE, 'r') as f:
assets = json.loads(f.read())
data_dir = default_data_directory()
coingecko = Coingecko(data_directory=data_dir)
COINGECKO_COINS_FILE = data_dir / 'coingecko.json'
if COINGECKO_COINS_FILE.exists():
with open(COINGECKO_COINS_FILE, 'r') as f:
coingecko_coins = json.loads(f.read())
else:
coingecko_coins = coingecko.all_coins()
with open(COINGECKO_COINS_FILE, 'w') as f:
f.write(rlk_jsondumps(coingecko_coins))
coingecko_add = {
'FTT': 'farmatrust',
'SNX': 'synthetix-network-token',
'0xBTC': '0xbitcoin',
def main():
arg_parser = aggregator_args()
args = arg_parser.parse_args()
msg_aggregator = MessagesAggregator()
user_data_dir = Path(default_data_directory()) / args.db_user
database = DBHandler(
user_data_dir=user_data_dir,
password=args.db_password,
msg_aggregator=msg_aggregator,
)
our_data = AssetResolver().assets
paprika = CoinPaprika()
cmc = None
cmc_list = None
root_path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.realpath(__file__))))
data_directory = f'{Path.home()}/.rotkehlchen'
if args.cmc_api_key:
cmc = Coinmarketcap(
data_directory=data_directory,
api_key=args.cmc_api_key,
)
cmc_list = cmc.get_cryptocyrrency_map()
cryptocompare = Cryptocompare(data_directory=data_directory,
database=database)
paprika_coins_list = paprika.get_coins_list()
cryptocompare_coins_map = cryptocompare.all_coins()
if args.input_file:
if not os.path.isfile(args.input_file):
print(f'Given input file {args.input_file} is not a file')
sys.exit(1)
with open(args.input_file, 'r') as f:
input_data = rlk_jsonloads(f.read())
given_symbols = set(input_data.keys())
current_symbols = set(our_data.keys())
if not given_symbols.isdisjoint(current_symbols):
print(
f'The following given input symbols already exist in the '
f'all_assets.json file {given_symbols.intersection(current_symbols)}',
)
sys.exit(1)
# If an input file is given, iterate only its assets and perform checks
for asset_symbol in input_data.keys():
input_data = process_asset(
our_data=input_data,
asset_symbol=asset_symbol,
paprika_coins_list=paprika_coins_list,
paprika=paprika,
cmc_list=cmc_list,
cryptocompare_coins_map=cryptocompare_coins_map,
always_keep_our_time=args.always_keep_our_time,
)
# and now combine the two dictionaries to get the final one. Note that no
# checks are perfomed for what was in all_assets.json before the script
# ran in this case
our_data = {**our_data, **input_data}
else:
# Iterate all of the assets of the all_assets.json file and perform checks
for asset_symbol in our_data.keys():
our_data = process_asset(
our_data=our_data,
asset_symbol=asset_symbol,
paprika_coins_list=paprika_coins_list,
paprika=paprika,
cmc_list=cmc_list,
cryptocompare_coins_map=cryptocompare_coins_map,
always_keep_our_time=args.always_keep_our_time,
)
# Finally overwrite the all_assets.json with the modified assets
with open(
os.path.join(root_path, 'rotkehlchen', 'data', 'all_assets.json'),
'w') as f:
f.write(json.dumps(
our_data,
sort_keys=True,
indent=4,
), )
import datetime
import re
import sys
from pathlib import Path
import py
import pytest
from rotkehlchen.config import default_data_directory
from rotkehlchen.globaldb.handler import GlobalDBHandler
GlobalDBHandler(default_data_directory())
from rotkehlchen.tests.fixtures import * # noqa: F401,F403
# monkey patch web3's non-thread safe lru cache with our own version
from rotkehlchen.chain.ethereum import patch_web3 # isort:skip # pylint: disable=unused-import # lgtm[py/unused-import] # noqa
assert sys.version_info.major == 3, 'Need to use python 3 for rotki'
assert 6 <= sys.version_info.minor <= 7, 'Need to use python 3.6 or python 3.7 for rotki'
def pytest_addoption(parser):
parser.addoption(
'--initial-port',
type=int,
default=29870,
help=
'Base port number used to avoid conflicts while running parallel tests.',
)
parser.addoption('--profiler', default=None, choices=['flamegraph-trace'])
def app_args(prog: str, description: str) -> argparse.ArgumentParser:
"""Add the Rotki arguments to the argument parser and return it"""
p = argparse.ArgumentParser(
prog=prog,
description=description,
)
p.add_argument(
'--sleep-secs',
type=int,
default=20,
help="Seconds to sleep during the main loop",
)
p.add_argument(
'--data-dir',
help='The directory where all data and configs are placed',
default=default_data_directory(),
)
p.add_argument(
'--api-host',
help='The host on which the rest API will run',
default='127.0.0.1',
)
p.add_argument(
'--api-port',
help='The port on which the rest API will run',
type=int,
default=5042,
)
p.add_argument(
'--api-cors',
help=
'Comma separated list of domains for the API to accept cross origin requests.',
default="http://localhost:*/*",
type=str,
)
p.add_argument(
'--ethrpc-port',
help="The port on which to communicate with an ethereum client's RPC.",
default=8545,
)
p.add_argument(
'--logfile',
help='The name of the file to write log entries to',
default='rotkehlchen.log',
)
p.add_argument(
'--logtarget',
help=
'Choose where logging entries will be sent. Valid values are "file and "stdout"',
choices=['stdout', 'file'],
default='file',
)
p.add_argument(
'--loglevel',
help='Choose the logging level',
choices=['debug', 'info', 'warn', 'error', 'critical'],
default='debug',
)
p.add_argument(
'--logfromothermodules',
help=('If given then logs from all imported modules that use the '
'logging system will also be visible.'),
action='store_true',
)
p.add_argument(
'version',
help='Shows the rotkehlchen version',
action=CommandAction,
)
return p
def find_coingecko_by_id(identifier: str, coins):
for coingecko_entry in coins:
if coingecko_entry['id'] == identifier:
return coingecko_entry
return None
root_dir = Path(__file__).resolve().parent.parent.parent
ASSETS_FILE = Path(f'{root_dir}/rotkehlchen/data/all_assets.json')
with open(ASSETS_FILE, 'r') as f:
assets = json.loads(f.read())
coingecko = Coingecko()
COINGECKO_COINS_FILE = default_data_directory() / 'coingecko.json'
if COINGECKO_COINS_FILE.exists():
with open(COINGECKO_COINS_FILE, 'r') as f:
coingecko_coins = json.loads(f.read())
else:
coingecko_coins = coingecko.all_coins()
with open(COINGECKO_COINS_FILE, 'w') as f:
f.write(rlk_jsondumps(coingecko_coins))
coingecko_add = {
'FTT': 'farmatrust',
'SNX': 'synthetix-network-token',
'0xBTC': '0xbitcoin',
'1SG': '1sg',
'1ST': 'first-blood',
def __init__(self) -> None:
self.id_to_variable: Dict[str, str] = {}
self.globaldb = GlobalDBHandler(default_data_directory())
def __init__(self, args: argparse.Namespace) -> None:
"""Initialize the Rotkehlchen object
May Raise:
- SystemPermissionError if the given data directory's permissions
are not correct.
"""
self.lock = Semaphore()
self.lock.acquire()
# Can also be None after unlock if premium credentials did not
# authenticate or premium server temporarily offline
self.premium: Optional[Premium] = None
self.user_is_logged_in = False
logfilename = None
if args.logtarget == 'file':
logfilename = args.logfile
if args.loglevel == 'debug':
loglevel = logging.DEBUG
elif args.loglevel == 'info':
loglevel = logging.INFO
elif args.loglevel == 'warn':
loglevel = logging.WARN
elif args.loglevel == 'error':
loglevel = logging.ERROR
elif args.loglevel == 'critical':
loglevel = logging.CRITICAL
else:
raise AssertionError('Should never get here. Illegal log value')
logging.basicConfig(
filename=logfilename,
filemode='w',
level=loglevel,
format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s',
datefmt='%d/%m/%Y %H:%M:%S %Z',
)
if not args.logfromothermodules:
logging.getLogger('urllib3').setLevel(logging.CRITICAL)
logging.getLogger('urllib3.connectionpool').setLevel(
logging.CRITICAL)
self.sleep_secs = args.sleep_secs
if args.data_dir is None:
self.data_dir = default_data_directory()
else:
self.data_dir = Path(args.data_dir)
if not os.access(self.data_dir, os.W_OK | os.R_OK):
raise SystemPermissionError(
f'The given data directory {self.data_dir} is not readable or writable',
)
self.args = args
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
self.exchange_manager = ExchangeManager(
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.data = DataHandler(self.data_dir, self.msg_aggregator)
self.cryptocompare = Cryptocompare(data_directory=self.data_dir,
database=None)
# Initialize the Inquirer singleton
Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare)
self.lock.release()
self.shutdown_event = gevent.event.Event()
