def price_historian(
data_dir,
inquirer, # pylint: disable=unused-argument
should_mock_price_queries,
mocked_price_queries,
cryptocompare,
session_coingecko,
default_mock_price_value,
historical_price_oracles_order,
dont_mock_price_for,
):
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
historian = PriceHistorian(
data_directory=data_dir,
cryptocompare=cryptocompare,
coingecko=session_coingecko,
)
historian.set_oracles_order(historical_price_oracles_order)
maybe_mock_historical_price_queries(
historian=historian,
should_mock_price_queries=should_mock_price_queries,
mocked_price_queries=mocked_price_queries,
default_mock_value=default_mock_price_value,
dont_mock_price_for=dont_mock_price_for,
)
return historian
def fixture_fake_price_historian(historical_price_oracles_order):
# NB: custom fixture for quick unit testing. Do not export.
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
price_historian = PriceHistorian(
data_directory=MagicMock(spec=Path),
cryptocompare=MagicMock(spec=Cryptocompare),
coingecko=MagicMock(spec=Coingecko),
)
price_historian.set_oracles_order(historical_price_oracles_order)
return price_historian
class BuchfinkDB(DBHandler):
"""
This class is not very thought out and might need a refactor. Currently it
does three things, namely:
1) preparing classes from Rotki to be used by higher-level functions
2) function as a Rotki DBHandler and provide data to Rotki classes
3) load and parse Buchfink config
"""
def __init__(self, data_directory='.'):
# pylint: disable=super-init-not-called
self.data_directory = Path(data_directory)
with open(self.data_directory / 'buchfink.yaml', 'r') as cfg:
yaml_config = yaml.load(cfg, Loader=yaml.SafeLoader)
self.config = config_schema(yaml_config)
self.accounts = accounts_from_config(self.config) # type: List[Account]
self._active_eth_address = None # type: Optional[ChecksumEthAddress]
self.reports_directory = self.data_directory / "reports"
self.trades_directory = self.data_directory / "trades"
self.cache_directory = self.data_directory / "cache"
self.balances_directory = self.data_directory / "balances"
self.annotations_directory = self.data_directory / "annotations"
self.user_data_dir = self.data_directory / "user"
self.reports_directory.mkdir(exist_ok=True)
self.trades_directory.mkdir(exist_ok=True)
self.balances_directory.mkdir(exist_ok=True)
self.cache_directory.mkdir(exist_ok=True)
(self.cache_directory / 'cryptocompare').mkdir(exist_ok=True)
(self.cache_directory / 'history').mkdir(exist_ok=True)
(self.cache_directory / 'inquirer').mkdir(exist_ok=True)
(self.cache_directory / 'coingecko').mkdir(exist_ok=True)
self.last_write_ts: Optional[Timestamp] = None
self._amm_swaps = [] # type: List[AMMSwap]
self._eth_tx = [] # type: List[EthereumTransaction]
self._eth_receipts_store = pickledb.load(self.cache_directory / 'receipts.db', False)
self.cryptocompare = Cryptocompare(self.cache_directory / 'cryptocompare', self)
self.coingecko = Coingecko()
self.historian = PriceHistorian(
self.cache_directory / 'history',
self.cryptocompare,
self.coingecko
)
self.inquirer = Inquirer(self.cache_directory / 'inquirer',
self.cryptocompare,
self.coingecko
)
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(msg_aggregator=self.msg_aggregator)
# Initialize blockchain querying modules
self.etherscan = Etherscan(database=self, msg_aggregator=self.msg_aggregator)
GlobalDBHandler._GlobalDBHandler__instance = None
self.globaldb = GlobalDBHandler(self.cache_directory)
self.asset_resolver = AssetResolver()
self.assets_updater = AssetsUpdater(self.msg_aggregator)
self.ethereum_manager = EthereumManager(
database=self,
ethrpc_endpoint=self.get_eth_rpc_endpoint(),
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=[]
)
self.inquirer.inject_ethereum(self.ethereum_manager)
self.inquirer.set_oracles_order(self.get_settings().current_price_oracles)
self.historian.set_oracles_order(self.get_settings().historical_price_oracles)
self.beaconchain = BeaconChain(database=self, msg_aggregator=self.msg_aggregator)
def __del__(self):
pass
def get_asset_by_symbol(self, symbol: str) -> Asset:
# TODO: this indirection function could incorporate a custom mapping from yaml config
return deserialize_asset(symbol)
def get_main_currency(self):
return self.get_settings().main_currency
def get_eth_rpc_endpoint(self):
return self.config['settings'].get('eth_rpc_endpoint', None)
def get_all_accounts(self) -> List[Account]:
return self.accounts
def get_all_reports(self) -> Iterable[ReportConfig]:
for report_info in self.config['reports']:
yield ReportConfig(
name=str(report_info['name']),
title=report_info.get('title'),
template=report_info.get('template'),
from_dt=datetime.fromisoformat(str(report_info['from'])),
to_dt=datetime.fromisoformat(str(report_info['to']))
)
def get_settings(self, have_premium: bool = False) -> DBSettings:
clean_settings = dict(self.config['settings'])
if 'external_services' in clean_settings:
del clean_settings['external_services']
return db_settings_from_dict(clean_settings, self.msg_aggregator)
def get_ignored_assets(self):
return []
def get_external_service_credentials(
self,
service_name: ExternalService,
) -> Optional[ExternalServiceApiCredentials]:
"""If existing it returns the external service credentials for the given service"""
short_name = service_name.name.lower()
api_key = self.config['settings'].get('external_services', {}).get(short_name)
if not api_key:
return None
return ExternalServiceApiCredentials(service=service_name, api_key=api_key)
def get_accountant(self) -> Accountant:
return Accountant(self, None, self.msg_aggregator, True, premium=None)
def get_blockchain_accounts(self) -> BlockchainAccounts:
if self._active_eth_address:
return BlockchainAccounts(eth=[self._active_eth_address], btc=[], ksm=[])
return BlockchainAccounts(eth=[], btc=[], ksm=[])
def get_trades_from_file(self, trades_file) -> List[Trade]:
def safe_deserialize_trade(trade):
try:
return deserialize_trade(trade)
except UnknownAsset:
logger.warning('Ignoring trade with unknown asset: %s', trade)
return None
exchange = yaml.load(open(trades_file, 'r'), Loader=yaml.SafeLoader)
return [ser_trade
for ser_trade in [
safe_deserialize_trade(trade) for trade in exchange.get('trades', [])]
if ser_trade is not None] \
+ [ser_trade
for ser_trade in [
safe_deserialize_trade(trade) for trade in exchange.get('actions', [])
if 'buy' in trade or 'sell' in trade]
if ser_trade is not None]
def get_local_trades_for_account(self, account_name: Union[str, Account]) -> List[Trade]:
if isinstance(account_name, str):
account = [a for a in self.accounts if a.name == account_name][0] # type: Account
else:
account = account_name
if account.account_type == 'file':
trades_file = os.path.join(self.data_directory, account.config['file'])
return self.get_trades_from_file(trades_file)
trades_file = os.path.join(self.data_directory, 'trades', account.name + '.yaml')
if os.path.exists(trades_file):
return self.get_trades_from_file(trades_file)
return []
def get_actions_from_file(self, actions_file):
def safe_deserialize_ledger_action(action):
if 'buy' in action or 'sell' in action:
return None
try:
return deserialize_ledger_action(action)
except UnknownAsset:
logger.warning('Ignoring ledger action with unknown asset: %s', action)
return None
exchange = yaml.load(open(actions_file, 'r'), Loader=yaml.SafeLoader)
return [ser_action
for ser_action in [
safe_deserialize_ledger_action(action)
for action in exchange.get('actions', [])
]
if ser_action is not None]
def get_local_ledger_actions_for_account(self, account_name: Union[str, Account]) \
-> List[Trade]:
if isinstance(account_name, str):
account = [a for a in self.accounts if a.name == account_name][0] # type: Account
else:
account = account_name
if account.account_type == 'file':
actions_file = self.data_directory / account.config['file']
if actions_file.exists():
return self.get_actions_from_file(actions_file)
else:
actions_file = self.data_directory / f'actions/{account.name}.yaml'
if actions_file.exists():
return self.get_actions_from_file(actions_file)
return []
def get_chain_manager(self, account: Account) -> ChainManager:
if account.account_type == "ethereum":
accounts = BlockchainAccounts(eth=[account.address], btc=[], ksm=[])
elif account.account_type == "bitcoin":
accounts = BlockchainAccounts(eth=[], btc=[account.address], ksm=[])
else:
raise ValueError('Unable to create chain manager for account')
# Eventually we should allow premium credentials in config file
premium = False
eth_modules = self.get_settings().active_modules
if not premium:
eth_modules = [mod for mod in eth_modules if mod not in PREMIUM_ONLY_ETH_MODULES]
logger.debug('Creating ChainManager with modules: %s', eth_modules)
manager = ChainManager(
database=self,
blockchain_accounts=accounts,
beaconchain=self.beaconchain,
data_directory=self.data_directory,
ethereum_manager=self.ethereum_manager,
kusama_manager=None,
msg_aggregator=self.msg_aggregator,
btc_derivation_gap_limit=self.get_settings().btc_derivation_gap_limit,
greenlet_manager=self.greenlet_manager,
premium=premium,
eth_modules=eth_modules
)
# Monkey-patch function that uses singleton
manager.queried_addresses_for_module = lambda self, module = None: [account]
return manager
def get_exchange(self, account: str) -> ExchangeInterface:
account_info = [a for a in self.config['accounts'] if a['name'] == account][0]
exchange_opts = dict(
name=account_info['name'],
api_key=str(account_info['api_key']),
secret=str(account_info['secret']).encode(),
database=self,
msg_aggregator=self.msg_aggregator
)
if account_info['exchange'] == 'kraken':
exchange = Kraken(**exchange_opts)
elif account_info['exchange'] == 'binance':
exchange = Binance(**exchange_opts)
elif account_info['exchange'] == 'coinbase':
exchange = Coinbase(**exchange_opts)
elif account_info['exchange'] == 'coinbasepro':
exchange = Coinbasepro(**exchange_opts, passphrase=str(account_info['passphrase']))
elif account_info['exchange'] == 'gemini':
exchange = Gemini(**exchange_opts)
elif account_info['exchange'] == 'bitmex':
exchange = Bitmex(**exchange_opts)
elif account_info['exchange'] == 'bittrex':
exchange = Bittrex(**exchange_opts)
elif account_info['exchange'] == 'poloniex':
exchange = Poloniex(**exchange_opts)
elif account_info['exchange'] == 'bitcoinde':
exchange = Bitcoinde(**exchange_opts)
elif account_info['exchange'] == 'iconomi':
exchange = Iconomi(**exchange_opts)
else:
raise ValueError("Unknown exchange: " + account_info['exchange'])
return exchange
def get_tokens_for_address_if_time(self, address, current_time):
return None
def save_tokens_for_address(self, address, tokens):
pass
def query_balances(self, account) -> BalanceSheet:
if account.account_type == "exchange":
exchange = self.get_exchange(account.name)
api_key_is_valid, error = exchange.validate_api_key()
if not api_key_is_valid:
raise RuntimeError(error)
balances, error = exchange.query_balances()
if not error:
logger.info(
'Fetched balances for %d assets from %s',
len(balances.keys()),
account.name
)
return BalanceSheet(assets=balances, liabilities={})
raise RuntimeError(error)
if account.account_type == "ethereum":
manager = self.get_chain_manager(account)
# This is a little hack because query_balances sometimes hooks back
# into out get_blockchain_accounts() without providing context (for
# example from makerdao module).
self._active_eth_address = account.address
manager.query_balances(
blockchain=SupportedBlockchain.ETHEREUM,
force_token_detection=True
)
self._active_eth_address = None
return reduce(operator.add, manager.balances.eth.values())
if account.account_type == "bitcoin":
manager = self.get_chain_manager(account)
manager.query_balances()
btc = Asset('BTC')
return BalanceSheet(assets={
btc: reduce(operator.add, manager.balances.btc.values())
}, liabilities={})
if account.account_type == "file":
return self.get_balances_from_file(account.config['file'])
return BalanceSheet(assets={}, liabilities={})
def fetch_balances(self, account):
query_sheet = self.query_balances(account)
path = self.annotations_directory / (account.name + '.yaml')
if path.exists():
query_sheet += self.get_balances_from_file(path)
self.write_balances(account, query_sheet)
def get_balances(self, account) -> BalanceSheet:
path = self.balances_directory / (account.name + '.yaml')
if path.exists():
return self.get_balances_from_file(path)
return BalanceSheet(assets={}, liabilities={})
def get_balances_from_file(self, path) -> BalanceSheet:
account = yaml.load(open(path, 'r'), Loader=yaml.SafeLoader)
assets = {} # type: Dict[Asset, Balance]
liabilities = {} # type: Dict[Asset, Balance]
if 'balances' in account:
logger.warning('Found deprecated key "balances", please use "assets" instead.')
for balance in account['balances']:
balance, asset = deserialize_balance(balance, self)
if asset in assets:
assets[asset] += balance
else:
assets[asset] = balance
if 'assets' in account:
for balance in account['assets']:
balance, asset = deserialize_balance(balance, self)
if asset in assets:
assets[asset] += balance
else:
assets[asset] = balance
if 'liabilities' in account:
for balance in account['liabilities']:
balance, asset = deserialize_balance(balance, self)
if asset in liabilities:
liabilities[asset] += balance
else:
liabilities[asset] = balance
return BalanceSheet(assets=assets, liabilities=liabilities)
def write_balances(self, account: Account, balances: BalanceSheet):
path = self.balances_directory / (account.name + '.yaml')
with path.open('w') as balances_file:
yaml.dump(serialize_balances(balances), stream=balances_file)
def get_amm_swaps(
self,
from_ts: Optional[Timestamp] = None,
to_ts: Optional[Timestamp] = None,
location: Optional[Location] = None,
address: Optional[ChecksumEthAddress] = None,
) -> List[AMMSwap]:
return self._amm_swaps
def add_amm_swaps(self, swaps: List[AMMSwap]) -> None:
self._amm_swaps = []
self._amm_swaps.extend(swaps)
def update_used_query_range(self, name: str, start_ts: Timestamp, end_ts: Timestamp) -> None:
pass
def update_used_block_query_range(self, name: str, from_block: int, to_block: int) -> None:
pass
def get_used_query_range(self, name: str) -> Optional[Tuple[Timestamp, Timestamp]]:
return None
def get_ethereum_transaction_receipt(self, tx_hash: str, manager: ChainManager):
receipt = self._eth_receipts_store.get(tx_hash)
if receipt:
return receipt
receipt = manager.ethereum.get_transaction_receipt(tx_hash)
self._eth_receipts_store.set(tx_hash, receipt)
self._eth_receipts_store.dump()
return receipt
def get_ignored_action_ids(
self,
action_type: Optional[ActionType],
) -> Dict[ActionType, List[str]]:
return {}
def add_ethereum_transactions(
self,
ethereum_transactions: List[EthereumTransaction],
from_etherscan: bool,
) -> None:
self._eth_tx = []
self._eth_tx.extend(ethereum_transactions)
def get_ethereum_transactions(
self,
from_ts: Optional[Timestamp] = None,
to_ts: Optional[Timestamp] = None,
address: Optional[ChecksumEthAddress] = None,
) -> List[EthereumTransaction]:
return self._eth_tx
def perform_assets_updates(self):
self.assets_updater.perform_update(None, 'remote')
for token_data in self.config.get('tokens', []):
eth_token = deserialize_ethereum_token(token_data)
identifier = '_ceth_' + eth_token.address
try:
self.get_asset_by_symbol(identifier)
logger.debug('Asset already exists: %s', eth_token)
except UnknownAsset:
self.globaldb.add_asset(identifier, AssetType.ETHEREUM_TOKEN, eth_token)
try:
self.get_asset_by_symbol(identifier)
except UnknownAsset as exc:
raise ValueError('Unable to add asset: ' + str(eth_token)) from exc
self.asset_resolver.clean_memory_cache()
