def fixture_data_dir(use_clean_caching_directory, tmpdir_factory) -> Path: """The tests data dir is peristent so that we can cache price queries between tests. If use_clean_caching_directory is True then a completely fresh dir is returned""" if use_clean_caching_directory: return Path(tmpdir_factory.mktemp('test_data_dir')) if 'CI' in os.environ: data_directory = Path.home() / '.cache' / '.rotkehlchen-test-dir' else: data_directory = default_data_directory().parent / 'test_data' data_directory.mkdir(parents=True, exist_ok=True) # do not keep pull github assets between tests. Can really confuse test results # as we may end up with different set of assets in tests (data_directory / 'assets').unlink(missing_ok=True) # Remove any old accounts. The only reason we keep this directory around is for # cached price queries, not for user DBs for x in data_directory.iterdir(): directory_with_db = (x.is_dir() and (x / 'rotkehlchen.db').exists() or (x / 'rotkehlchen_transient.db').exists()) if directory_with_db: shutil.rmtree(x, ignore_errors=True) return data_directory
def __init__(self, args: argparse.Namespace) -> None: self.db = DBHandler( user_data_dir=default_data_directory() / args.user_name, password=args.user_password, msg_aggregator=MessagesAggregator(), initial_settings=None, )
def app_args(prog: str, description: str) -> argparse.ArgumentParser: """Add the Rotkehlchen arguments to the argument parser and return it""" p = argparse.ArgumentParser( prog=prog, description=description, ) p.add_argument( '--sleep-secs', type=int, default=20, help="Seconds to sleep during the main loop", ) p.add_argument( '--data-dir', help='The directory where all data and configs are placed', default=default_data_directory(), ) p.add_argument( '--zerorpc-port', help= 'The port on which to open a zerorpc server for communication with the UI', default=4242, ) p.add_argument( '--ethrpc-port', help="The port on which to communicate with an ethereum client's RPC.", default=8545, ) p.add_argument( '--logfile', help='The name of the file to write log entries to', default='rotkehlchen.log', ) p.add_argument( '--logtarget', help= 'Choose where logging entries will be sent. Valid values are "file and "stdout"', choices=['stdout', 'file'], default='file', ) p.add_argument( '--loglevel', help='Choose the logging level', choices=['debug', 'info', 'warn', 'error', 'critical'], default='debug', ) p.add_argument( '--logfromothermodules', help=('If given then logs from all imported modules that use the ' 'logging system will also be visible.'), action='store_true', ) p.add_argument( 'version', help='Shows the rotkehlchen version', action=VersionAction, ) return p
def __init__(self, args: argparse.Namespace) -> None: user_path = FilePath( os.path.join(str(default_data_directory()), args.user_name)) self.db = DBHandler( user_data_dir=user_path, password=args.user_password, msg_aggregator=MessagesAggregator(), )
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager(msg_aggregator=self.msg_aggregator) # Initialize the AssetResolver singleton AssetResolver(data_directory=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.greenlet_manager.spawn_and_track( after_seconds=None, task_name='periodically_query_icons_until_all_cached', method=self.icon_manager.periodically_query_icons_until_all_cached, batch_size=ICONS_BATCH_SIZE, sleep_time_secs=ICONS_QUERY_SLEEP, ) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.lock.release() self.shutdown_event = gevent.event.Event()
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object This runs during backend initialization so it should be as light as possible. May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) self.data_dir.mkdir(parents=True, exist_ok=True) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.main_loop_spawned = False self.args = args self.api_task_greenlets: List[gevent.Greenlet] = [] self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) # Initialize the GlobalDBHandler singleton. Has to be initialized BEFORE asset resolver GlobalDBHandler(data_dir=self.data_dir) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) self.coingecko = Coingecko() self.icon_manager = IconManager(data_dir=self.data_dir, coingecko=self.coingecko) self.assets_updater = AssetsUpdater(self.msg_aggregator) # Initialize the Inquirer singleton Inquirer( data_dir=self.data_dir, cryptocompare=self.cryptocompare, coingecko=self.coingecko, ) # Keeps how many trades we have found per location. Used for free user limiting self.actions_per_location: Dict[str, Dict[Location, int]] = { 'trade': defaultdict(int), 'asset_movement': defaultdict(int), } self.task_manager: Optional[TaskManager] = None self.shutdown_event = gevent.event.Event()
def app_args() -> argparse.Namespace: """ Parse the arguments and create and return the arguments object""" p = argparse.ArgumentParser( description='Rotkehlchen Crypto Portfolio Management') p.add_argument( '--output', help=('A path to a file for logging all output. If nothing is given' 'stdout is used')) p.add_argument('--sleep-secs', type=int, default=20, help="Seconds to sleep during the main loop") p.add_argument('--notify', action='store_true', help=('If given then the tool will send notifications via ' 'notify-send.')) p.add_argument('--data-dir', help='The directory where all data and configs are placed', default=default_data_directory()) p.add_argument( '--zerorpc-port', help= 'The port on which to open a zerorpc server for communication with the UI', default=4242) p.add_argument( '--ethrpc-port', help="The port on which to communicate with an ethereum client's RPC.", default=8545, ) p.add_argument( '--logfile', help='The name of the file to write log entries to', default='rotkehlchen.log', ) p.add_argument( '--logtarget', help= 'Choose where logging entries will be sent. Valid values are "file and "stdout"', choices=['stdout', 'file'], default='file', ) p.add_argument('--loglevel', help='Choose the logging level', choices=['debug', 'info', 'warn', 'error', 'critical'], default='debug') p.add_argument( '--logfromothermodules', help=('If given then logs from all imported modules that use the ' 'logging system will also be visible.'), action='store_true', ) args = p.parse_args() return args
def data_dir(use_clean_caching_directory, tmpdir_factory) -> Path: """The tests data dir is peristent so that we can cache price queries between tests. If use_clean_caching_directory is True then a completely fresh dir is returned""" if use_clean_caching_directory: return Path(tmpdir_factory.mktemp('test_data_dir')) if 'CI' in os.environ: data_directory = Path.home() / '.cache' / '.rotkehlchen-test-dir' else: data_directory = default_data_directory().parent / 'test_data' data_directory.mkdir(parents=True, exist_ok=True) # Remove any old accounts. The only reason we keep this directory around is for # cached price queries, not for user DBs for x in data_directory.iterdir(): if x.is_dir() and (x / 'rotkehlchen.db').exists(): shutil.rmtree(x) return data_directory
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in: bool = False configure_logging(args) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event()
def find_coingecko_by_id(identifier: str, coins): for coingecko_entry in coins: if coingecko_entry['id'] == identifier: return coingecko_entry return None root_dir = Path(__file__).resolve().parent.parent.parent ASSETS_FILE = Path(f'{root_dir}/rotkehlchen/data/all_assets.json') with open(ASSETS_FILE, 'r') as f: assets = json.loads(f.read()) data_dir = default_data_directory() coingecko = Coingecko(data_directory=data_dir) COINGECKO_COINS_FILE = data_dir / 'coingecko.json' if COINGECKO_COINS_FILE.exists(): with open(COINGECKO_COINS_FILE, 'r') as f: coingecko_coins = json.loads(f.read()) else: coingecko_coins = coingecko.all_coins() with open(COINGECKO_COINS_FILE, 'w') as f: f.write(rlk_jsondumps(coingecko_coins)) coingecko_add = { 'FTT': 'farmatrust', 'SNX': 'synthetix-network-token', '0xBTC': '0xbitcoin',
def main(): arg_parser = aggregator_args() args = arg_parser.parse_args() msg_aggregator = MessagesAggregator() user_data_dir = Path(default_data_directory()) / args.db_user database = DBHandler( user_data_dir=user_data_dir, password=args.db_password, msg_aggregator=msg_aggregator, ) our_data = AssetResolver().assets paprika = CoinPaprika() cmc = None cmc_list = None root_path = os.path.dirname( os.path.dirname(os.path.dirname(os.path.realpath(__file__)))) data_directory = f'{Path.home()}/.rotkehlchen' if args.cmc_api_key: cmc = Coinmarketcap( data_directory=data_directory, api_key=args.cmc_api_key, ) cmc_list = cmc.get_cryptocyrrency_map() cryptocompare = Cryptocompare(data_directory=data_directory, database=database) paprika_coins_list = paprika.get_coins_list() cryptocompare_coins_map = cryptocompare.all_coins() if args.input_file: if not os.path.isfile(args.input_file): print(f'Given input file {args.input_file} is not a file') sys.exit(1) with open(args.input_file, 'r') as f: input_data = rlk_jsonloads(f.read()) given_symbols = set(input_data.keys()) current_symbols = set(our_data.keys()) if not given_symbols.isdisjoint(current_symbols): print( f'The following given input symbols already exist in the ' f'all_assets.json file {given_symbols.intersection(current_symbols)}', ) sys.exit(1) # If an input file is given, iterate only its assets and perform checks for asset_symbol in input_data.keys(): input_data = process_asset( our_data=input_data, asset_symbol=asset_symbol, paprika_coins_list=paprika_coins_list, paprika=paprika, cmc_list=cmc_list, cryptocompare_coins_map=cryptocompare_coins_map, always_keep_our_time=args.always_keep_our_time, ) # and now combine the two dictionaries to get the final one. Note that no # checks are perfomed for what was in all_assets.json before the script # ran in this case our_data = {**our_data, **input_data} else: # Iterate all of the assets of the all_assets.json file and perform checks for asset_symbol in our_data.keys(): our_data = process_asset( our_data=our_data, asset_symbol=asset_symbol, paprika_coins_list=paprika_coins_list, paprika=paprika, cmc_list=cmc_list, cryptocompare_coins_map=cryptocompare_coins_map, always_keep_our_time=args.always_keep_our_time, ) # Finally overwrite the all_assets.json with the modified assets with open( os.path.join(root_path, 'rotkehlchen', 'data', 'all_assets.json'), 'w') as f: f.write(json.dumps( our_data, sort_keys=True, indent=4, ), )
import datetime import re import sys from pathlib import Path import py import pytest from rotkehlchen.config import default_data_directory from rotkehlchen.globaldb.handler import GlobalDBHandler GlobalDBHandler(default_data_directory()) from rotkehlchen.tests.fixtures import * # noqa: F401,F403 # monkey patch web3's non-thread safe lru cache with our own version from rotkehlchen.chain.ethereum import patch_web3 # isort:skip # pylint: disable=unused-import # lgtm[py/unused-import] # noqa assert sys.version_info.major == 3, 'Need to use python 3 for rotki' assert 6 <= sys.version_info.minor <= 7, 'Need to use python 3.6 or python 3.7 for rotki' def pytest_addoption(parser): parser.addoption( '--initial-port', type=int, default=29870, help= 'Base port number used to avoid conflicts while running parallel tests.', ) parser.addoption('--profiler', default=None, choices=['flamegraph-trace'])
def app_args(prog: str, description: str) -> argparse.ArgumentParser: """Add the Rotki arguments to the argument parser and return it""" p = argparse.ArgumentParser( prog=prog, description=description, ) p.add_argument( '--sleep-secs', type=int, default=20, help="Seconds to sleep during the main loop", ) p.add_argument( '--data-dir', help='The directory where all data and configs are placed', default=default_data_directory(), ) p.add_argument( '--api-host', help='The host on which the rest API will run', default='127.0.0.1', ) p.add_argument( '--api-port', help='The port on which the rest API will run', type=int, default=5042, ) p.add_argument( '--api-cors', help= 'Comma separated list of domains for the API to accept cross origin requests.', default="http://localhost:*/*", type=str, ) p.add_argument( '--ethrpc-port', help="The port on which to communicate with an ethereum client's RPC.", default=8545, ) p.add_argument( '--logfile', help='The name of the file to write log entries to', default='rotkehlchen.log', ) p.add_argument( '--logtarget', help= 'Choose where logging entries will be sent. Valid values are "file and "stdout"', choices=['stdout', 'file'], default='file', ) p.add_argument( '--loglevel', help='Choose the logging level', choices=['debug', 'info', 'warn', 'error', 'critical'], default='debug', ) p.add_argument( '--logfromothermodules', help=('If given then logs from all imported modules that use the ' 'logging system will also be visible.'), action='store_true', ) p.add_argument( 'version', help='Shows the rotkehlchen version', action=CommandAction, ) return p
def find_coingecko_by_id(identifier: str, coins): for coingecko_entry in coins: if coingecko_entry['id'] == identifier: return coingecko_entry return None root_dir = Path(__file__).resolve().parent.parent.parent ASSETS_FILE = Path(f'{root_dir}/rotkehlchen/data/all_assets.json') with open(ASSETS_FILE, 'r') as f: assets = json.loads(f.read()) coingecko = Coingecko() COINGECKO_COINS_FILE = default_data_directory() / 'coingecko.json' if COINGECKO_COINS_FILE.exists(): with open(COINGECKO_COINS_FILE, 'r') as f: coingecko_coins = json.loads(f.read()) else: coingecko_coins = coingecko.all_coins() with open(COINGECKO_COINS_FILE, 'w') as f: f.write(rlk_jsondumps(coingecko_coins)) coingecko_add = { 'FTT': 'farmatrust', 'SNX': 'synthetix-network-token', '0xBTC': '0xbitcoin', '1SG': '1sg', '1ST': 'first-blood',
def __init__(self) -> None: self.id_to_variable: Dict[str, str] = {} self.globaldb = GlobalDBHandler(default_data_directory())
def __init__(self, args: argparse.Namespace) -> None: """Initialize the Rotkehlchen object May Raise: - SystemPermissionError if the given data directory's permissions are not correct. """ self.lock = Semaphore() self.lock.acquire() # Can also be None after unlock if premium credentials did not # authenticate or premium server temporarily offline self.premium: Optional[Premium] = None self.user_is_logged_in = False logfilename = None if args.logtarget == 'file': logfilename = args.logfile if args.loglevel == 'debug': loglevel = logging.DEBUG elif args.loglevel == 'info': loglevel = logging.INFO elif args.loglevel == 'warn': loglevel = logging.WARN elif args.loglevel == 'error': loglevel = logging.ERROR elif args.loglevel == 'critical': loglevel = logging.CRITICAL else: raise AssertionError('Should never get here. Illegal log value') logging.basicConfig( filename=logfilename, filemode='w', level=loglevel, format='%(asctime)s -- %(levelname)s:%(name)s:%(message)s', datefmt='%d/%m/%Y %H:%M:%S %Z', ) if not args.logfromothermodules: logging.getLogger('urllib3').setLevel(logging.CRITICAL) logging.getLogger('urllib3.connectionpool').setLevel( logging.CRITICAL) self.sleep_secs = args.sleep_secs if args.data_dir is None: self.data_dir = default_data_directory() else: self.data_dir = Path(args.data_dir) if not os.access(self.data_dir, os.W_OK | os.R_OK): raise SystemPermissionError( f'The given data directory {self.data_dir} is not readable or writable', ) self.args = args self.msg_aggregator = MessagesAggregator() self.greenlet_manager = GreenletManager( msg_aggregator=self.msg_aggregator) self.exchange_manager = ExchangeManager( msg_aggregator=self.msg_aggregator) self.all_eth_tokens = AssetResolver().get_all_eth_tokens() self.data = DataHandler(self.data_dir, self.msg_aggregator) self.cryptocompare = Cryptocompare(data_directory=self.data_dir, database=None) # Initialize the Inquirer singleton Inquirer(data_dir=self.data_dir, cryptocompare=self.cryptocompare) self.lock.release() self.shutdown_event = gevent.event.Event()