class Accountant():
def __init__(
self,
db: DBHandler,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory, create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = Timestamp(-1)
self.currently_processing_timestamp = Timestamp(-1)
def __del__(self) -> None:
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def _customize(self, settings: DBSettings) -> None:
"""Customize parameters after pulling DBSettings"""
if settings.include_crypto2crypto is not None:
self.events.include_crypto2crypto = settings.include_crypto2crypto
if settings.taxfree_after_period is not None:
given_taxfree_after_period: Optional[int] = settings.taxfree_after_period
if given_taxfree_after_period == -1:
# That means user requested to disable taxfree_after_period
given_taxfree_after_period = None
self.events.taxfree_after_period = given_taxfree_after_period
self.profit_currency = settings.main_currency
self.events.profit_currency = settings.main_currency
self.csvexporter.profit_currency = settings.main_currency
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
"""
Adds the given asset movement to the processed events
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
timestamp = movement.timestamp
if timestamp < self.start_ts:
return
if movement.asset.identifier == 'KFEE':
# There is no reason to process deposits of KFEE for kraken as it has only value
# internal to kraken and KFEE has no value and will error at cryptocompare price query
return
fee_rate = self.events.get_rate_in_profit_currency(movement.fee_asset, timestamp)
cost = movement.fee * fee_rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=movement.category,
asset=movement.asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=movement.location,
)
self.csvexporter.add_asset_movement(
exchange=movement.location,
category=movement.category,
asset=movement.asset,
fee=movement.fee,
rate=fee_rate,
timestamp=timestamp,
)
def account_for_gas_costs(
self,
transaction: EthereumTransaction,
include_gas_costs: bool,
) -> None:
"""
Accounts for the gas costs of the given ethereum transaction
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if not include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.events.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10 ** 18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.tx_hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade, loan_settlement: bool) -> None:
"""
Adds the given trade to the sell events
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.events.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Union[Trade, MarginPosition]],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
# Used only in the "avoid zerorpc remote lost after 10ms problem"
self.last_sleep_ts = 0
# Ask the DB for the settings once at the start of processing so we got the
# same settings through the entire task
db_settings = self.db.get_settings()
self._customize(db_settings)
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(
key=lambda action: action_get_timestamp(action),
)
# The first ts is the ts of the first action we have in history or 0 for empty history
first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(actions[0])
self.currently_processing_timestamp = first_ts
self.started_processing_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count, db_settings)
except PriceQueryUnknownFromAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}',
)
continue
except RemoteError as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to reach an external '
f'service at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to reach an external service at that time: {str(e)}',
)
continue
if not should_continue:
break
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (
self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs
)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit': str(self.events.loan_profit),
'margin_positions_profit_loss': str(self.events.margin_positions_profit_loss),
'settlement_losses': str(self.events.settlement_losses),
'ethereum_transaction_gas_costs': str(self.eth_transactions_gas_costs),
'asset_movement_fees': str(self.asset_movement_fees),
'general_trade_profit_loss': str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss': str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss': str(total_taxable_pl),
'total_profit_loss': str(
self.events.general_trade_profit_loss +
sum_other_actions,
),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
db_settings: DBSettings,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.',
)
return True, prev_time, count
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant():
def __init__(
self,
db: DBHandler,
user_directory: Path,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(
database=db,
user_directory=user_directory,
create_csv=create_csv,
)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = 0
self.currently_processing_timestamp = -1
self.first_processed_timestamp = -1
def __del__(self) -> None:
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def _customize(self, settings: DBSettings) -> None:
"""Customize parameters after pulling DBSettings"""
if settings.include_crypto2crypto is not None:
self.events.include_crypto2crypto = settings.include_crypto2crypto
if settings.taxfree_after_period is not None:
given_taxfree_after_period: Optional[
int] = settings.taxfree_after_period
if given_taxfree_after_period == -1:
# That means user requested to disable taxfree_after_period
given_taxfree_after_period = None
self.events.taxfree_after_period = given_taxfree_after_period
self.profit_currency = settings.main_currency
self.events.profit_currency = settings.main_currency
self.csvexporter.profit_currency = settings.main_currency
if settings.account_for_assets_movements is not None:
self.events.account_for_assets_movements = settings.account_for_assets_movements
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
"""
Adds the given asset movement to the processed events
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
timestamp = movement.timestamp
if timestamp < self.start_ts:
return
if movement.asset.identifier == 'KFEE' or not self.events.account_for_assets_movements:
# There is no reason to process deposits of KFEE for kraken as it has only value
# internal to kraken and KFEE has no value and will error at cryptocompare price query
return
fee_rate = self.events.get_rate_in_profit_currency(
movement.fee_asset, timestamp)
cost = movement.fee * fee_rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=movement.category,
asset=movement.asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=movement.location,
)
self.csvexporter.add_asset_movement(
exchange=movement.location,
category=movement.category,
asset=movement.asset,
fee=movement.fee,
rate=fee_rate,
timestamp=timestamp,
)
def account_for_gas_costs(
self,
transaction: EthereumTransaction,
include_gas_costs: bool,
) -> None:
"""
Accounts for the gas costs of the given ethereum transaction
May raise:
- PriceQueryUnsupportedAsset if from/to assets are missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if not include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.events.get_rate_in_profit_currency(A_ETH,
transaction.timestamp)
eth_burned_as_gas = FVal(transaction.gas_used * gas_price) / FVal(10**
18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.tx_hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
"""
Adds the given trade to the sell events
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.events.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
location=trade.location,
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
location=trade.location,
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Union[Trade, MarginPosition, AMMTrade]],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
defi_events: List[DefiEvent],
ledger_actions: List[LedgerAction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = 2000000000
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset()
# Ask the DB for the settings once at the start of processing so we got the
# same settings through the entire task
db_settings = self.db.get_settings()
self._customize(db_settings)
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
if len(defi_events) != 0:
actions.extend(defi_events)
if len(ledger_actions) != 0:
actions.extend(ledger_actions)
actions.sort(key=action_get_timestamp)
# The first ts is the ts of the first action we have in history or 0 for empty history
first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(
actions[0])
self.currently_processing_timestamp = first_ts
self.first_processed_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
ignored_actionids_mapping = self.db.get_ignored_action_ids(
action_type=None)
for action in actions:
try:
(
should_continue,
prev_time,
) = self._process_action(
action=action,
start_ts=start_ts,
end_ts=end_ts,
prev_time=prev_time,
db_settings=db_settings,
ignored_actionids_mapping=ignored_actionids_mapping,
)
except PriceQueryUnsupportedAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details', )
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}', )
continue
except RemoteError as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to inability to reach an external '
f'service at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to reach an external service at that time: {str(e)}',
)
continue
if not should_continue:
break
if count % 500 == 0:
# This loop can take a very long time depending on the amount of actions
# to process. We need to yield to other greenlets or else calls to the
# API may time out
gevent.sleep(0.5)
count += 1
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.defi_profit_loss +
self.events.ledger_actions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'ledger_actions_profit_loss':
str(self.events.ledger_actions_profit_loss),
'defi_profit_loss':
str(self.events.defi_profit_loss),
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
@staticmethod
def _should_ignore_action(
action: TaxableAction,
action_type: str,
ignored_actionids_mapping: Dict[ActionType, List[str]],
) -> Tuple[bool, Any]:
# TODO: These ifs/mappings of action type str to the enum
# are only due to mix of new and old code. They should be removed and only
# the enum should be used everywhere eventually
should_ignore = False
identifier: Optional[Any] = None
if action_type == 'trade':
trade = cast(Trade, action)
identifier = trade.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.TRADE, [])
elif action_type == 'asset_movement':
movement = cast(AssetMovement, action)
identifier = movement.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.ASSET_MOVEMENT,
[],
)
elif action_type == 'ethereum_transaction':
tx = cast(EthereumTransaction, action)
identifier = tx.identifier
should_ignore = tx.identifier in ignored_actionids_mapping.get(
ActionType.ETHEREUM_TX,
[],
)
elif action_type == 'ledger_action':
ledger_action = cast(LedgerAction, action)
identifier = ledger_action.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.LEDGER_ACTION,
[],
)
return should_ignore, identifier
def _process_action(
self,
action: TaxableAction,
start_ts: Timestamp,
end_ts: Timestamp,
prev_time: Timestamp,
db_settings: DBSettings,
ignored_actionids_mapping: Dict[ActionType, List[str]],
) -> Tuple[bool, Timestamp]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if not db_settings.calculate_past_cost_basis and timestamp < start_ts:
# ignore older actions than start_ts if we don't want past cost basis
return True, prev_time
if timestamp > end_ts:
# reached the end of the time period for the report
return False, prev_time
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except DeserializationError:
self.msg_aggregator.add_error(
'At history processing found trade with non string asset type. '
'Ignoring the trade.', )
return True, prev_time
if isinstance(asset1, UnknownEthereumToken) or isinstance(
asset2, UnknownEthereumToken): # type: ignore # noqa: E501
# TODO: Typing needs fixing here # type: ignore
log.debug( # type: ignore
'Ignoring action with unknown token',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
should_ignore, identifier = self._should_ignore_action(
action=action,
action_type=action_type,
ignored_actionids_mapping=ignored_actionids_mapping,
)
if should_ignore:
log.info(
f'Ignoring {action_type} action with identifier {identifier} '
f'at {timestamp} since the user asked to ignore it', )
return True, prev_time
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
location=action.location,
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time
if action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time
if action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time
if action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time
if action_type == 'defi_event':
action = cast(DefiEvent, action)
self.events.add_defi_event(action)
return True, prev_time
if action_type == 'ledger_action':
action = cast(LedgerAction, action)
self.events.add_ledger_action(action)
return True, prev_time
# else if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
location=trade.location,
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
location=trade.location,
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant():
def __init__(
self,
profit_currency: Asset,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.set_main_currency(profit_currency.identifier)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = -1
self.currently_processing_timestamp = -1
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __del__(self):
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
if 'include_crypto2crypto' in settings:
given_include_c2c = settings['include_crypto2crypto']
if not isinstance(given_include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
self.events.include_crypto2crypto = given_include_c2c
if 'taxfree_after_period' in settings:
given_taxfree_after_period = settings['taxfree_after_period']
if given_taxfree_after_period is not None:
if not isinstance(given_taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if given_taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
given_taxfree_after_period = given_taxfree_after_period * 86400
settings['taxfree_after_period'] = given_taxfree_after_period
self.events.taxfree_after_period = given_taxfree_after_period
return True, ''
def set_main_currency(self, given_currency: str) -> None:
currency = Asset(given_currency)
msg = 'main currency checks should have happened at rotkehlchen.set_settings()'
assert currency.is_fiat(), msg
self.profit_currency = currency
self.events.profit_currency = currency
@staticmethod
def query_historical_price(
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> FVal:
price = PriceHistorian().query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = self.query_historical_price(
asset,
self.profit_currency,
timestamp,
)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
fee_rate = self.query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return fee_rate * trade.fee
def add_asset_movement_to_events(
self,
category: str,
asset: Asset,
timestamp: Timestamp,
exchange: Exchange,
fee: Fee,
) -> None:
if timestamp < self.start_ts:
return
rate = self.get_rate_in_profit_currency(asset, timestamp)
cost = fee * rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=category,
asset=asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=exchange,
)
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
if not self.include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Trade],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
# Used only in the "avoid zerorpc remote lost after 10ms problem"
self.last_sleep_ts = 0
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action), )
# The first timestamp is the timestamp of the first action we have in history
first_ts = action_get_timestamp(actions[0])
self.currently_processing_timestamp = first_ts
self.started_processing_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count)
except PriceQueryUnknownFromAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details', )
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}', )
continue
if not should_continue:
break
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not"""
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client. (after 10s)
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely be async
now = ts_now()
if now - self.last_sleep_ts >= 7: # choose 7 seconds to be safe
self.last_sleep_ts = now
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.', )
return True, prev_time, count
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(
category=action.category,
asset=action.asset,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee,
)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(
gain_loss_asset=action.pl_currency,
gain_loss_amount=action.profit_loss,
fee_in_asset=Fee(ZERO),
margin_notes=action.notes,
timestamp=action.close_time,
)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
raise ValueError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant(object):
def __init__(
self,
price_historian: PriceHistorian,
profit_currency: FiatAsset,
user_directory: FilePath,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __del__(self):
del self.events
del self.csvexporter
del self.price_historian
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
include_c2c = self.events.include_crypto2crypto
taxfree_after_period = self.events.taxfree_after_period
if 'include_crypto2crypto' in settings:
include_c2c = settings['include_crypto2crypto']
if not isinstance(include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
if 'taxfree_after_period' in settings:
taxfree_after_period = settings['taxfree_after_period']
if taxfree_after_period is not None:
if not isinstance(taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
taxfree_after_period = taxfree_after_period * 86400
settings['taxfree_after_period'] = taxfree_after_period
self.events.include_crypto2crypto = include_c2c
self.events.taxfree_after_period = taxfree_after_period
return True, ''
def set_main_currency(self, currency: FiatAsset) -> None:
if currency not in FIAT_CURRENCIES:
raise ValueError(
'Attempted to set unsupported "{}" as main currency.'.format(
currency), )
self.profit_currency = currency
self.events.profit_currency = currency
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> FVal:
price = self.price_historian.query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = self.query_historical_price(
asset,
self.profit_currency,
timestamp,
)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
fee_rate = self.query_historical_price(
trade.fee_currency,
self.profit_currency,
trade.timestamp,
)
return fee_rate * trade.fee
def add_asset_movement_to_events(
self,
category: str,
asset: Asset,
amount: FVal,
timestamp: Timestamp,
exchange: str,
fee: Fee,
) -> None:
if timestamp < self.start_ts:
return
rate = self.get_rate_in_profit_currency(asset, timestamp)
cost = fee * rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=category,
asset=asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=exchange,
)
if category == 'withdrawal':
assert fee != 0, 'So far all exchanges charge you for withdrawing'
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
if not self.include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency(S_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
selling_asset = trade_get_other_pair(trade, trade.cost_currency)
selling_asset_rate = self.get_rate_in_profit_currency(
trade.cost_currency,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=trade.cost_currency,
receiving_amount=trade.cost,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Trade],
margin_history: List[Trade],
loan_history: Dict,
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(margin_history) != 0:
actions.extend(margin_history)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action), )
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count)
except PriceQueryUnknownFromAsset as e:
log.error(
f'Skipping trade during history processing: {str(e)}')
continue
if not should_continue:
break
self.events.calculate_asset_details()
self.price_historian.inquirer.save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not"""
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client.
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely by async
count += 1
if count % 500 == 0:
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
action_type = action_get_type(action)
asset1, asset2 = action_get_assets(action)
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Dict, action)
self.events.add_loan_gain(
gained_asset=action['currency'],
lent_amount=action['amount_lent'],
gained_amount=action['earned'],
fee_in_asset=action['fee'],
open_time=action['open_time'],
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(
category=action.category,
asset=action.asset,
amount=action.amount,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee,
)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(
gain_loss_asset=action.pl_currency,
gain_loss_amount=action.profit_loss,
fee_in_asset=Fee(ZERO),
margin_notes=action.notes,
timestamp=action.close_time,
)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# if the cost is not equal to rate * amount then the data is somehow corrupt
if not trade.cost.is_close(trade.amount * trade.rate, max_diff="1e-4"):
raise CorruptData(
"Trade found with cost {} which is not equal to trade.amount"
"({}) * trade.rate({})".format(trade.cost, trade.amount,
trade.rate), )
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.type == 'buy':
self.events.add_buy_and_corresponding_sell(
bought_asset=trade_get_other_pair(trade, trade.cost_currency),
bought_amount=trade.amount,
paid_with_asset=trade.cost_currency,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.type == 'sell':
self.trade_add_to_sell_events(trade, False)
elif trade.type == 'settlement_sell':
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.type == 'settlement_buy':
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = S_BTC
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
selling_amount = trade.cost
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
raise ValueError('Unknown trade type "{}" encountered'.format(
trade.type))
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant(object):
def __init__(
self,
price_historian,
profit_currency,
user_directory,
create_csv,
ignored_assets,
include_crypto2crypto,
taxfree_after_period,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
# Customizable Options
self.ignored_assets = ignored_assets
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
@property
def general_trade_pl(self):
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self):
return self.events.taxable_trade_profit_loss
def customize(self, settings):
include_c2c = self.events.include_crypto2crypto
taxfree_after_period = self.events.taxfree_after_period
if 'include_crypto2crypto' in settings:
include_c2c = settings['include_crypto2crypto']
if not isinstance(include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
if 'taxfree_after_period' in settings:
taxfree_after_period = settings['taxfree_after_period']
if taxfree_after_period is not None:
if not isinstance(taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
taxfree_after_period = taxfree_after_period * 86400
settings['taxfree_after_period'] = taxfree_after_period
self.events.include_crypto2crypto = include_c2c
self.events.taxfree_after_period = taxfree_after_period
return True, ''
def set_main_currency(self, currency):
if currency not in FIAT_CURRENCIES:
raise ValueError(
'Attempted to set unsupported "{}" as main currency.'.format(
currency))
self.profit_currency = currency
self.events.profit_currency = currency
def query_historical_price(self, from_asset, to_asset, timestamp):
price = self.price_historian.query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset, timestamp):
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = 1
else:
rate = self.query_historical_price(asset, self.profit_currency,
timestamp)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def add_asset_movement_to_events(self, category, asset, amount, timestamp,
exchange, fee):
rate = self.get_rate_in_profit_currency(asset, timestamp)
self.asset_movement_fees += fee * rate
if category == 'withdrawal':
assert fee != 0, "So far all exchanges charge you for withdrawing"
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction):
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency('ETH', transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
self.eth_transactions_gas_costs += eth_burned_as_gas * rate
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade, loan_settlement):
selling_asset = trade_get_other_pair(trade, trade.cost_currency)
selling_asset_rate = self.get_rate_in_profit_currency(
trade.cost_currency, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=trade.cost_currency,
receiving_amount=trade.cost,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False)
def process_history(self, start_ts, end_ts, trade_history, margin_history,
loan_history, asset_movements, eth_transactions):
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
"""
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
actions = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(margin_history) != 0:
actions.extend(margin_history)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action))
prev_time = 0
count = 0
for action in actions:
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client.
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely by async
count += 1
if count % 500 == 0:
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
break
action_type = action_get_type(action)
asset1, asset2 = action_get_assets(action)
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Ignoring {} with {} {}".format(
action_type, asset1, asset2))
continue
if action_type == 'loan':
self.events.add_loan_gain(
gained_asset=action['currency'],
lent_amount=action['amount_lent'],
gained_amount=action['earned'],
fee_in_asset=action['fee'],
open_time=action['open_time'],
close_time=timestamp,
)
continue
elif action_type == 'asset_movement':
self.add_asset_movement_to_events(category=action.category,
asset=action.asset,
amount=action.amount,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee)
continue
elif action_type == 'margin_position':
self.events.add_margin_position(
gained_asset='BTC',
gained_amount=action['btc_profit_loss'],
fee_in_asset=0,
margin_notes=action['notes'],
timestamp=timestamp)
continue
elif action_type == 'ethereum_transaction':
self.account_for_gas_costs(action)
continue
# if we get here it's a trade
trade = action
# if the cost is not equal to rate * amount then the data is somehow corrupt
if not trade.cost.is_close(trade.amount * trade.rate,
max_diff="1e-5"):
raise CorruptData(
"Trade found with cost {} which is not equal to trade.amount"
"({}) * trade.rate({})".format(trade.cost, trade.amount,
trade.rate))
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.type == 'buy':
self.events.add_buy_and_corresponding_sell(
bought_asset=trade_get_other_pair(trade,
trade.cost_currency),
bought_amount=trade.amount,
paid_with_asset=trade.cost_currency,
trade_rate=trade.rate,
trade_fee=trade.fee,
fee_currency=trade.fee_currency,
timestamp=trade.timestamp)
elif trade.type == 'sell':
self.trade_add_to_sell_events(trade, False)
elif trade.type == 'settlement_sell':
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.type == 'settlement_buy':
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = 'BTC'
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.cost,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True)
else:
raise ValueError('Unknown trade type "{}" encountered'.format(
trade.type))
self.events.calculate_asset_details()
sum_other_actions = (self.events.margin_positions_profit +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit':
str(self.events.margin_positions_profit),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss + sum_other_actions),
},
'all_events': self.csvexporter.all_events,
}