def price_historian(
accounting_data_dir,
inquirer, # pylint: disable=unused-argument
should_mock_price_queries,
mocked_price_queries,
cryptocompare,
):
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
historian = PriceHistorian(
data_directory=accounting_data_dir,
history_date_start=TEST_HISTORY_DATA_START,
cryptocompare=cryptocompare,
)
if should_mock_price_queries:
def mock_historical_price_query(from_asset, to_asset, timestamp):
if from_asset == to_asset:
return FVal(1)
try:
price = mocked_price_queries[from_asset.identifier][
to_asset.identifier][timestamp]
except KeyError:
raise AssertionError(
f'No mocked price found from {from_asset.identifier} to '
f'{to_asset.identifier} at {timestamp}', )
return price
historian.query_historical_price = mock_historical_price_query
return historian
def price_historian(
data_dir,
inquirer, # pylint: disable=unused-argument
should_mock_price_queries,
mocked_price_queries,
cryptocompare,
session_coingecko,
default_mock_price_value,
historical_price_oracles_order,
):
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
historian = PriceHistorian(
data_directory=data_dir,
cryptocompare=cryptocompare,
coingecko=session_coingecko,
)
historian.set_oracles_order(historical_price_oracles_order)
maybe_mock_historical_price_queries(
historian=historian,
should_mock_price_queries=should_mock_price_queries,
mocked_price_queries=mocked_price_queries,
default_mock_value=default_mock_price_value,
)
return historian
def _adjust_to_cryptocompare_price_incosistencies(
price: Price,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> Price:
"""Doublecheck against the USD rate, and if incosistencies are found
then take the USD adjusted price.
This is due to incosistencies in the provided historical data from
cryptocompare. https://github.com/rotki/rotki/issues/221
Note: Since 12/01/2019 this seems to no longer be happening, but I will
keep the code around just in case a regression is introduced on the side
of cryptocompare.
May raise:
- PriceQueryUnsupportedAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
from_asset_usd = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_USD,
timestamp=timestamp,
)
to_asset_usd = PriceHistorian().query_historical_price(
from_asset=to_asset,
to_asset=A_USD,
timestamp=timestamp,
)
usd_invert_conversion = Price(from_asset_usd / to_asset_usd)
abs_diff = abs(usd_invert_conversion - price)
relative_difference = abs_diff / max(price, usd_invert_conversion)
if relative_difference >= FVal('0.1'):
log.warning(
'Cryptocompare historical price data are incosistent.'
'Taking USD adjusted price. Check github issue #221',
from_asset=from_asset,
to_asset=to_asset,
incosistent_price=price,
usd_price=from_asset_usd,
adjusted_price=usd_invert_conversion,
)
return usd_invert_conversion
return price
def set_settings(self, settings: ModifiableDBSettings) -> Tuple[bool, str]:
"""Tries to set new settings. Returns True in success or False with message if error"""
with self.lock:
if settings.eth_rpc_endpoint is not None:
result, msg = self.chain_manager.set_eth_rpc_endpoint(
settings.eth_rpc_endpoint)
if not result:
return False, msg
if settings.ksm_rpc_endpoint is not None:
result, msg = self.chain_manager.set_ksm_rpc_endpoint(
settings.ksm_rpc_endpoint)
if not result:
return False, msg
if settings.kraken_account_type is not None:
kraken = self.exchange_manager.get('kraken')
if kraken:
kraken.set_account_type(
settings.kraken_account_type) # type: ignore
if settings.btc_derivation_gap_limit is not None:
self.chain_manager.btc_derivation_gap_limit = settings.btc_derivation_gap_limit
if settings.current_price_oracles is not None:
Inquirer().set_oracles_order(settings.current_price_oracles)
if settings.historical_price_oracles is not None:
PriceHistorian().set_oracles_order(
settings.historical_price_oracles)
self.data.db.set_settings(settings)
return True, ''
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
# unlock or create the DB
self.password = password
user_dir = self.data.unlock(user, password, create_new)
self.try_premium_at_start(api_key, api_secret, create_new,
sync_approval, user_dir)
secret_data = self.data.db.get_exchange_secrets()
self.cache_data_filename = os.path.join(self.data_dir,
'cache_data.json')
historical_data_start = self.data.historical_start_date()
self.trades_historian = TradesHistorian(
self.data_dir,
self.data.db,
self.data.get_eth_accounts(),
historical_data_start,
)
self.price_historian = PriceHistorian(
self.data_dir,
historical_data_start,
)
self.accountant = Accountant(price_historian=self.price_historian,
profit_currency=self.data.main_currency(),
create_csv=True)
self.inquirer = Inquirer(kraken=self.kraken)
self.initialize_exchanges(secret_data)
self.blockchain = Blockchain(self.data.db.get_blockchain_accounts(),
self.data.eth_tokens,
self.data.db.get_owned_tokens(),
self.inquirer, self.args.ethrpc_port)
def query_historical_price(from_asset: Asset, to_asset: Asset,
timestamp: Timestamp):
return PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
def query_historical_price(
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> FVal:
price = PriceHistorian().query_historical_price(from_asset, to_asset, timestamp)
return price
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
create_new=create_new,
sync_approval=sync_approval,
)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
historical_data_start=historical_data_start,
)
self.inquirer = Inquirer(data_dir=self.data_dir, kraken=self.kraken)
price_historian = PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
inquirer=self.inquirer,
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
price_historian=price_historian,
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'],
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
all_eth_tokens=self.data.eth_tokens,
owned_eth_tokens=self.data.db.get_owned_tokens(),
inquirer=self.inquirer,
ethchain=ethchain,
)
def __init__(self, data_directory='.'):
self.data_directory = Path(data_directory)
self.config = yaml.load(open(self.data_directory / 'buchfink.yaml',
'r'),
Loader=yaml.SafeLoader)
self.reports_directory = self.data_directory / "reports"
self.trades_directory = self.data_directory / "trades"
self.cache_directory = self.data_directory / "cache"
self.reports_directory.mkdir(exist_ok=True)
self.trades_directory.mkdir(exist_ok=True)
self.cache_directory.mkdir(exist_ok=True)
(self.cache_directory / 'cryptocompare').mkdir(exist_ok=True)
(self.cache_directory / 'history').mkdir(exist_ok=True)
(self.cache_directory / 'inquirer').mkdir(exist_ok=True)
self.cryptocompare = Cryptocompare(
self.cache_directory / 'cryptocompare', self)
self.historian = PriceHistorian(self.cache_directory / 'history',
'01/01/2014', self.cryptocompare)
self.inquirer = Inquirer(self.cache_directory / 'inquirer',
self.cryptocompare)
self.msg_aggregator = MessagesAggregator()
self.greenlet_manager = GreenletManager(
msg_aggregator=self.msg_aggregator)
# Initialize blockchain querying modules
self.etherscan = Etherscan(database=self,
msg_aggregator=self.msg_aggregator)
self.all_eth_tokens = AssetResolver().get_all_eth_tokens()
self.alethio = Alethio(
database=self,
msg_aggregator=self.msg_aggregator,
all_eth_tokens=self.all_eth_tokens,
)
self.ethereum_manager = EthereumManager(
ethrpc_endpoint=self.get_eth_rpc_endpoint(),
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
)
#self.chain_manager = ChainManager(
# blockchain_accounts=[],
# owned_eth_tokens=[],
# ethereum_manager=self.ethereum_manager,
# msg_aggregator=self.msg_aggregator,
# alethio=alethio,
# greenlet_manager=self.greenlet_manager,
# premium=False,
# eth_modules=ethereum_modules,
#)
self.ethereum_analyzer = EthereumAnalyzer(
ethereum_manager=self.ethereum_manager,
database=self,
)
def _adjust_to_cryptocompare_price_incosistencies(
price: Price,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> Price:
"""Doublecheck against the USD rate, and if incosistencies are found
then take the USD adjusted price.
This is due to incosistencies in the provided historical data from
cryptocompare. https://github.com/rotkehlchenio/rotkehlchen/issues/221
Note: Since 12/01/2019 this seems to no longer be happening, but I will
keep the code around just in case a regression is introduced on the side
of cryptocompare.
"""
from_asset_usd = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_USD,
timestamp=timestamp,
)
to_asset_usd = PriceHistorian().query_historical_price(
from_asset=to_asset,
to_asset=A_USD,
timestamp=timestamp,
)
usd_invert_conversion = from_asset_usd / to_asset_usd
abs_diff = abs(usd_invert_conversion - price)
relative_difference = abs_diff / max(price, usd_invert_conversion)
if relative_difference >= FVal('0.1'):
log.warning(
'Cryptocompare historical price data are incosistent.'
'Taking USD adjusted price. Check github issue #221',
from_asset=from_asset,
to_asset=to_asset,
incosistent_price=price,
usd_price=from_asset_usd,
adjusted_price=usd_invert_conversion,
)
return usd_invert_conversion
return price
def price_historian(
accounting_data_dir,
inquirer,
should_mock_price_queries,
mocked_price_queries,
):
historian = PriceHistorian(
data_directory=accounting_data_dir,
history_date_start=TEST_HISTORY_DATA_START,
inquirer=inquirer,
)
if should_mock_price_queries:
def mock_historical_price_query(from_asset, to_asset, timestamp):
if from_asset == to_asset:
return FVal(1)
return mocked_price_queries[from_asset][to_asset][timestamp]
historian.query_historical_price = mock_historical_price_query
return historian
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = PriceHistorian().query_historical_price(
from_asset=asset,
to_asset=self.profit_currency,
timestamp=timestamp,
)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def initialize_mock_rotkehlchen_instance(
rotki,
start_with_logged_in_user,
start_with_valid_premium,
db_password,
rotki_premium_credentials,
username,
blockchain_accounts,
owned_eth_tokens,
include_etherscan_key,
include_cryptocompare_key,
should_mock_price_queries,
mocked_price_queries,
ethereum_modules,
db_settings,
ignored_assets,
tags,
manually_tracked_balances,
default_mock_price_value,
):
if start_with_logged_in_user:
rotki.unlock_user(
user=username,
password=db_password,
create_new=True,
sync_approval='no',
premium_credentials=None,
given_ethereum_modules=ethereum_modules,
)
if start_with_valid_premium:
rotki.premium = Premium(rotki_premium_credentials)
rotki.premium_sync_manager.premium = rotki.premium
# After unlocking when all objects are created we need to also include
# customized fixtures that may have been set by the tests
rotki.chain_manager.owned_eth_tokens = owned_eth_tokens
rotki.chain_manager.accounts = blockchain_accounts
add_settings_to_test_db(rotki.data.db, db_settings, ignored_assets)
maybe_include_etherscan_key(rotki.data.db, include_etherscan_key)
maybe_include_cryptocompare_key(rotki.data.db,
include_cryptocompare_key)
add_blockchain_accounts_to_db(rotki.data.db, blockchain_accounts)
add_tags_to_test_db(rotki.data.db, tags)
add_manually_tracked_balances_to_test_db(rotki.data.db,
manually_tracked_balances)
maybe_mock_historical_price_queries(
historian=PriceHistorian(),
should_mock_price_queries=should_mock_price_queries,
mocked_price_queries=mocked_price_queries,
default_mock_value=default_mock_price_value,
)
def price_historian(
accounting_data_dir,
inquirer, # pylint: disable=unused-argument
should_mock_price_queries,
mocked_price_queries,
):
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
historian = PriceHistorian(
data_directory=accounting_data_dir,
history_date_start=TEST_HISTORY_DATA_START,
cryptocompare=Cryptocompare(data_directory=accounting_data_dir),
)
if should_mock_price_queries:
def mock_historical_price_query(from_asset, to_asset, timestamp):
if from_asset == to_asset:
return FVal(1)
return mocked_price_queries[from_asset][to_asset][timestamp]
historian.query_historical_price = mock_historical_price_query
return historian
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def init_accounting_tests(history_list, margin_list, start_ts, end_ts):
price_historian = PriceHistorian('/home/lefteris/.rotkehlchen')
accountant = Accountant(
price_historian=price_historian,
profit_currency='EUR',
create_csv=False
)
accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trades_from_dictlist(history_list, start_ts, end_ts),
margin_history=trades_from_dictlist(margin_list, start_ts, end_ts),
loan_history=list(),
asset_movements=list(),
eth_transactions=list()
)
return accountant
def init_accounting_tests(history_list, margin_list, start_ts, end_ts):
# TODO: This should become part of test fixtures. Also HAS to be deleted at teardown
user_directory = mkdtemp()
price_historian = PriceHistorian(user_directory)
accountant = Accountant(price_historian=price_historian,
profit_currency='EUR',
user_directory=user_directory,
create_csv=False,
ignored_assets=[])
accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trades_from_dictlist(history_list, start_ts, end_ts),
margin_history=trades_from_dictlist(margin_list, start_ts, end_ts),
loan_history=list(),
asset_movements=list(),
eth_transactions=list())
return accountant
def unlock_user(self, user, password, create_new, sync_approval, api_key,
api_secret):
# unlock or create the DB
self.password = password
user_dir = self.data.unlock(user, password, create_new)
self.try_premium_at_start(api_key, api_secret, create_new,
sync_approval, user_dir)
secret_data = self.data.db.get_exchange_secrets()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_port = settings['eth_rpc_port']
self.trades_historian = TradesHistorian(
self.data_dir,
self.data.db,
self.data.get_eth_accounts(),
historical_data_start,
)
self.price_historian = PriceHistorian(
self.data_dir,
historical_data_start,
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
price_historian=self.price_historian,
profit_currency=self.data.main_currency(),
user_directory=user_dir,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
)
self.inquirer = Inquirer(kraken=self.kraken)
self.initialize_exchanges(secret_data)
ethchain = Ethchain(eth_rpc_port)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
all_eth_tokens=self.data.eth_tokens,
owned_eth_tokens=self.data.db.get_owned_tokens(),
inquirer=self.inquirer,
ethchain=ethchain,
)
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
"""Get the profit_currency price of asset in the given timestamp
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = PriceHistorian().query_historical_price(
from_asset=asset,
to_asset=self.profit_currency,
timestamp=timestamp,
)
return rate
def price_historian(
data_dir,
inquirer, # pylint: disable=unused-argument
should_mock_price_queries,
mocked_price_queries,
cryptocompare,
):
# Since this is a singleton and we want it initialized everytime the fixture
# is called make sure its instance is always starting from scratch
PriceHistorian._PriceHistorian__instance = None
historian = PriceHistorian(
data_directory=data_dir,
history_date_start=TEST_HISTORY_DATA_START,
cryptocompare=cryptocompare,
)
maybe_mock_historical_price_queries(
historian=historian,
should_mock_price_queries=should_mock_price_queries,
mocked_price_queries=mocked_price_queries,
)
return historian
def _update_events_value(
self,
staking_events: ADXStakingEvents,
) -> None:
# Update amounts for unbonds and unbond requests
bond_id_bond_map: Dict[HexStr, Optional[Bond]] = {
bond.bond_id: bond
for bond in staking_events.bonds
}
for event in (
staking_events.unbonds + staking_events.
unbond_requests # type: ignore # mypy bug concatenating lists
):
has_bond = True
bond = bond_id_bond_map.get(event.bond_id, None)
if bond:
event.value = Balance(amount=bond.value.amount)
event.pool_id = bond.pool_id
elif event.bond_id not in bond_id_bond_map:
bond_id_bond_map[event.bond_id] = None
db_bonds = cast(
List[Bond],
self.database.get_adex_events(
bond_id=event.bond_id,
event_type=AdexEventType.BOND,
))
if db_bonds:
db_bond = db_bonds[0]
bond_id_bond_map[event.bond_id] = db_bond
event.value = Balance(amount=db_bond.value.amount)
event.pool_id = db_bond.pool_id
else:
has_bond = False
else:
has_bond = False
if has_bond is False:
msg = (f'Failed to update AdEx event data. '
f'Unable to find the related bond event: {event}')
self.msg_aggregator.add_error(msg)
# Update usd_value for all events
token_usd_price_history: Dict[Tuple[EthereumToken, Timestamp],
Price] = {}
for event in staking_events.get_all(
): # type: ignore # event can have all types
# NB: channel withdraw events can have nullable token
if isinstance(event, ChannelWithdraw):
if event.token is None:
continue
key = (event.token, event.timestamp)
token = event.token
else:
key = (A_ADX, event.timestamp)
token = A_ADX
usd_price = token_usd_price_history.get(key, None)
if usd_price is None:
usd_price = PriceHistorian().query_historical_price(
from_asset=token,
to_asset=A_USD,
timestamp=event.timestamp,
)
event.value.usd_value = event.value.amount * usd_price
token_usd_price_history[key] = usd_price
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: str,
premium_credentials: Optional[PremiumCredentials],
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
maybe_submit_usage_analytics(settings.submit_usage_analytics)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
alethio = Alethio(
database=self.data.db,
msg_aggregator=self.msg_aggregator,
all_eth_tokens=self.all_eth_tokens,
)
historical_data_start = settings.historical_data_start
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=self.cryptocompare,
)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethchain = Ethchain(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
)
makerdao = MakerDAO(
ethchain=ethchain,
database=self.data.db,
msg_aggregator=self.msg_aggregator,
)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
alethio=alethio,
greenlet_manager=self.greenlet_manager,
eth_modules={'makerdao': makerdao},
)
self.ethereum_analyzer = EthereumAnalyzer(
ethchain=ethchain,
database=self.data.db,
)
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.user_is_logged_in = True
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new,
initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
task_name='submit_usage_analytics',
method=maybe_submit_usage_analytics,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
historical_data_start = settings.historical_data_start
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=self.cryptocompare,
)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
)
self.ethereum_analyzer = EthereumAnalyzer(
ethereum_manager=ethereum_manager,
database=self.data.db,
)
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def price_historian(accounting_data_dir):
return PriceHistorian(accounting_data_dir, TEST_HISTORY_DATA_START)
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
May raise:
- PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
try:
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
except UnsupportedAsset as e:
raise PriceQueryUnsupportedAsset(e.asset_name)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is None or data[index].low is None:
# If we get some None in the hourly set price to 0 so that we check alternatives
price = Price(ZERO)
else:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
May raise:
- PriceQueryUnsupportedAsset if from/to asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
"""
try:
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
except UnsupportedAsset as e:
raise PriceQueryUnsupportedAsset(e.asset_name)
price = Price(ZERO)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index_in_bounds = True
# convert_to_int can't raise here due to its input
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
if index > len(data) - 1: # index out of bounds
# Try to see if index - 1 is there and if yes take it
if index > len(data):
index = index - 1
else: # give up. This happened: https://github.com/rotki/rotki/issues/1534
log.error(
f'Expected data index in cryptocompare historical hour price '
f'not found. Queried price of: {from_asset.identifier} in '
f'{to_asset.identifier} at {timestamp}. Data '
f'index: {index}. Length of returned data: {len(data)}. '
f'https://github.com/rotki/rotki/issues/1534. Attempting other methods...',
)
index_in_bounds = False
if index_in_bounds:
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is not None and data[index].low is not None:
price = Price((data[index].high + data[index].low) / 2)
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = Price(asset_btc_price * btc_to_asset_price)
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset=from_asset,
to_asset=to_asset,
date=timestamp_to_date(timestamp,
formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> Price:
if from_asset in KNOWN_TO_MISS_FROM_CRYPTOCOMPARE:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
historical_data_start=historical_data_start,
)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
if timestamp >= data[0].time:
index = convert_to_int((timestamp - data[0].time) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index].time - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1].time - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index].high is None or data[index].low is None:
# If we get some None in the hourly set price to 0 so that we check alternatives
price = Price(ZERO)
else:
price = (data[index].high + data[index].low) / 2
else:
# no price found in the historical data from/to asset, try alternatives
price = Price(ZERO)
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp}. Comparing with BTC...",
)
# Just get the BTC price
asset_btc_price = PriceHistorian().query_historical_price(
from_asset=from_asset,
to_asset=A_BTC,
timestamp=timestamp,
)
btc_to_asset_price = PriceHistorian().query_historical_price(
from_asset=A_BTC,
to_asset=to_asset,
timestamp=timestamp,
)
price = asset_btc_price * btc_to_asset_price
else:
log.debug(
f"Couldn't find historical price from {from_asset} to "
f"{to_asset} at timestamp {timestamp} through cryptocompare."
f" Attempting to get daily price...", )
price = self.query_endpoint_pricehistorical(
from_asset, to_asset, timestamp)
comparison_to_nonusd_fiat = ((to_asset.is_fiat() and to_asset != A_USD)
or (from_asset.is_fiat()
and from_asset != A_USD))
if comparison_to_nonusd_fiat:
price = self._adjust_to_cryptocompare_price_incosistencies(
price=price,
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset,
to_asset,
timestamp_to_date(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'),
)
log.debug(
'Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price,
)
return price
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: bool,
api_key: ApiKey,
api_secret: ApiSecret,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
api_key=api_key,
api_secret=api_secret,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except AuthenticationError:
# It means that our credentials were not accepted by the server
# or some other error happened
pass
exchange_credentials = self.data.db.get_exchange_credentials()
settings = self.data.db.get_settings()
historical_data_start = settings['historical_data_start']
eth_rpc_endpoint = settings['eth_rpc_endpoint']
self.trades_historian = TradesHistorian(
user_directory=self.user_directory,
db=self.data.db,
eth_accounts=self.data.get_eth_accounts(),
msg_aggregator=self.msg_aggregator,
)
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
history_date_start=historical_data_start,
cryptocompare=Cryptocompare(data_directory=self.data_dir),
)
db_settings = self.data.db.get_settings()
self.accountant = Accountant(
profit_currency=self.data.main_currency(),
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
ignored_assets=self.data.db.get_ignored_assets(),
include_crypto2crypto=db_settings['include_crypto2crypto'],
taxfree_after_period=db_settings['taxfree_after_period'],
include_gas_costs=db_settings['include_gas_costs'],
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=db_settings['anonymized_logs'])
self.initialize_exchanges(exchange_credentials)
ethchain = Ethchain(eth_rpc_endpoint)
self.blockchain = Blockchain(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
owned_eth_tokens=self.data.db.get_owned_tokens(),
ethchain=ethchain,
msg_aggregator=self.msg_aggregator,
)
self.user_is_logged_in = True
def unlock_user(
self,
user: str,
password: str,
create_new: bool,
sync_approval: Literal['yes', 'no', 'unknown'],
premium_credentials: Optional[PremiumCredentials],
initial_settings: Optional[ModifiableDBSettings] = None,
) -> None:
"""Unlocks an existing user or creates a new one if `create_new` is True
May raise:
- PremiumAuthenticationError if the password can't unlock the database.
- AuthenticationError if premium_credentials are given and are invalid
or can't authenticate with the server
- DBUpgradeError if the rotki DB version is newer than the software or
there is a DB upgrade and there is an error.
- SystemPermissionError if the directory or DB file can not be accessed
"""
log.info(
'Unlocking user',
user=user,
create_new=create_new,
sync_approval=sync_approval,
initial_settings=initial_settings,
)
# unlock or create the DB
self.password = password
self.user_directory = self.data.unlock(user, password, create_new,
initial_settings)
self.data_importer = DataImporter(db=self.data.db)
self.last_data_upload_ts = self.data.db.get_last_data_upload_ts()
self.premium_sync_manager = PremiumSyncManager(data=self.data,
password=password)
# set the DB in the external services instances that need it
self.cryptocompare.set_database(self.data.db)
# Anything that was set above here has to be cleaned in case of failure in the next step
# by reset_after_failed_account_creation_or_login()
try:
self.premium = self.premium_sync_manager.try_premium_at_start(
given_premium_credentials=premium_credentials,
username=user,
create_new=create_new,
sync_approval=sync_approval,
)
except PremiumAuthenticationError:
# Reraise it only if this is during the creation of a new account where
# the premium credentials were given by the user
if create_new:
raise
self.msg_aggregator.add_warning(
'Could not authenticate the Rotki premium API keys found in the DB.'
' Has your subscription expired?', )
# else let's just continue. User signed in succesfully, but he just
# has unauthenticable/invalid premium credentials remaining in his DB
settings = self.get_settings()
self.greenlet_manager.spawn_and_track(
after_seconds=None,
task_name='submit_usage_analytics',
exception_is_error=False,
method=maybe_submit_usage_analytics,
should_submit=settings.submit_usage_analytics,
)
self.etherscan = Etherscan(database=self.data.db,
msg_aggregator=self.msg_aggregator)
self.beaconchain = BeaconChain(database=self.data.db,
msg_aggregator=self.msg_aggregator)
eth_rpc_endpoint = settings.eth_rpc_endpoint
# Initialize the price historian singleton
PriceHistorian(
data_directory=self.data_dir,
cryptocompare=self.cryptocompare,
coingecko=self.coingecko,
)
PriceHistorian().set_oracles_order(settings.historical_price_oracles)
self.accountant = Accountant(
db=self.data.db,
user_directory=self.user_directory,
msg_aggregator=self.msg_aggregator,
create_csv=True,
)
# Initialize the rotkehlchen logger
LoggingSettings(anonymized_logs=settings.anonymized_logs)
exchange_credentials = self.data.db.get_exchange_credentials()
self.exchange_manager.initialize_exchanges(
exchange_credentials=exchange_credentials,
database=self.data.db,
)
# Initialize blockchain querying modules
ethereum_manager = EthereumManager(
ethrpc_endpoint=eth_rpc_endpoint,
etherscan=self.etherscan,
database=self.data.db,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=ETHEREUM_NODES_TO_CONNECT_AT_START,
)
kusama_manager = SubstrateManager(
chain=SubstrateChain.KUSAMA,
msg_aggregator=self.msg_aggregator,
greenlet_manager=self.greenlet_manager,
connect_at_start=KUSAMA_NODES_TO_CONNECT_AT_START,
connect_on_startup=self._connect_ksm_manager_on_startup(),
own_rpc_endpoint=settings.ksm_rpc_endpoint,
)
Inquirer().inject_ethereum(ethereum_manager)
Inquirer().set_oracles_order(settings.current_price_oracles)
self.chain_manager = ChainManager(
blockchain_accounts=self.data.db.get_blockchain_accounts(),
ethereum_manager=ethereum_manager,
kusama_manager=kusama_manager,
msg_aggregator=self.msg_aggregator,
database=self.data.db,
greenlet_manager=self.greenlet_manager,
premium=self.premium,
eth_modules=settings.active_modules,
data_directory=self.data_dir,
beaconchain=self.beaconchain,
btc_derivation_gap_limit=settings.btc_derivation_gap_limit,
)
self.events_historian = EventsHistorian(
user_directory=self.user_directory,
db=self.data.db,
msg_aggregator=self.msg_aggregator,
exchange_manager=self.exchange_manager,
chain_manager=self.chain_manager,
)
self.task_manager = TaskManager(
max_tasks_num=DEFAULT_MAX_TASKS_NUM,
greenlet_manager=self.greenlet_manager,
api_task_greenlets=self.api_task_greenlets,
database=self.data.db,
cryptocompare=self.cryptocompare,
premium_sync_manager=self.premium_sync_manager,
chain_manager=self.chain_manager,
exchange_manager=self.exchange_manager,
)
self.user_is_logged_in = True
log.debug('User unlocking complete')
def initialize_mock_rotkehlchen_instance(
rotki,
start_with_logged_in_user,
start_with_valid_premium,
db_password,
rotki_premium_credentials,
username,
blockchain_accounts,
include_etherscan_key,
include_cryptocompare_key,
should_mock_price_queries,
mocked_price_queries,
ethereum_modules,
db_settings,
ignored_assets,
tags,
manually_tracked_balances,
default_mock_price_value,
ethereum_manager_connect_at_start,
kusama_manager_connect_at_start,
eth_rpc_endpoint,
ksm_rpc_endpoint,
aave_use_graph,
max_tasks_num,
):
if not start_with_logged_in_user:
return
# Mock the initial get settings to include the specified ethereum modules
def mock_get_settings() -> DBSettings:
settings = DBSettings(
active_modules=ethereum_modules,
eth_rpc_endpoint=eth_rpc_endpoint,
ksm_rpc_endpoint=ksm_rpc_endpoint,
)
return settings
settings_patch = patch.object(rotki,
'get_settings',
side_effect=mock_get_settings)
# Do not connect to the usual nodes at start by default. Do not want to spam
# them during our tests. It's configurable per test, with the default being nothing
eth_rpcconnect_patch = patch(
'rotkehlchen.rotkehlchen.ETHEREUM_NODES_TO_CONNECT_AT_START',
new=ethereum_manager_connect_at_start,
)
ksm_rpcconnect_patch = patch(
'rotkehlchen.rotkehlchen.KUSAMA_NODES_TO_CONNECT_AT_START',
new=kusama_manager_connect_at_start,
)
ksm_connect_on_startup_patch = patch.object(
rotki,
'_connect_ksm_manager_on_startup',
return_value=bool(blockchain_accounts.ksm),
)
with settings_patch, eth_rpcconnect_patch, ksm_rpcconnect_patch, ksm_connect_on_startup_patch:
rotki.unlock_user(
user=username,
password=db_password,
create_new=True,
sync_approval='no',
premium_credentials=None,
)
# configure when task manager should run for tests
rotki.task_manager.max_tasks_num = max_tasks_num
if start_with_valid_premium:
rotki.premium = Premium(rotki_premium_credentials)
rotki.premium_sync_manager.premium = rotki.premium
# After unlocking when all objects are created we need to also include
# customized fixtures that may have been set by the tests
rotki.chain_manager.accounts = blockchain_accounts
add_settings_to_test_db(rotki.data.db, db_settings, ignored_assets)
maybe_include_etherscan_key(rotki.data.db, include_etherscan_key)
maybe_include_cryptocompare_key(rotki.data.db, include_cryptocompare_key)
add_blockchain_accounts_to_db(rotki.data.db, blockchain_accounts)
add_tags_to_test_db(rotki.data.db, tags)
add_manually_tracked_balances_to_test_db(rotki.data.db,
manually_tracked_balances)
maybe_mock_historical_price_queries(
historian=PriceHistorian(),
should_mock_price_queries=should_mock_price_queries,
mocked_price_queries=mocked_price_queries,
default_mock_value=default_mock_price_value,
)
wait_until_all_nodes_connected(
ethereum_manager_connect_at_start=ethereum_manager_connect_at_start,
ethereum=rotki.chain_manager.ethereum,
)
wait_until_all_substrate_nodes_connected(
substrate_manager_connect_at_start=kusama_manager_connect_at_start,
substrate_manager=rotki.chain_manager.kusama,
)
aave = rotki.chain_manager.get_module('aave')
if aave:
aave.use_graph = aave_use_graph