def __init__(
self,
price_historian: PriceHistorian,
profit_currency: FiatAsset,
user_directory: FilePath,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __init__(
self,
profit_currency: Asset,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.set_main_currency(profit_currency.identifier)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = -1
self.currently_processing_timestamp = -1
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __init__(self, price_historian, profit_currency, user_directory,
create_csv, ignored_assets):
self.price_historian = price_historian
self.set_main_currency(profit_currency)
self.ignored_assets = ignored_assets
# If this flag is True when your asset is being forcefully sold as a
# loan/margin settlement then profit/loss is also calculated before the entire
# amount is taken as a loss
self.count_profit_for_settlements = False
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
def __init__(
self,
db: DBHandler,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory, create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = Timestamp(-1)
self.currently_processing_timestamp = Timestamp(-1)
def __init__(
self,
price_historian,
profit_currency,
create_csv,
ignored_assets=[]):
self.price_historian = price_historian
self.set_main_currency(profit_currency)
self.ignored_assets = ignored_assets
# If this flag is True when your asset is being forcefully sold as a
# loan/margin settlement then profit/loss is also calculated before the entire
# amount is taken as a loss
self.count_profit_for_settlements = False
self.csvexporter = CSVExporter(profit_currency, create_csv)
# TEMPORARY FOR TESTING. TODO: Remove
self.temp_list = list()
def __init__(
self,
db: DBHandler,
user_directory: Path,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
log.debug('Initializing Accountant')
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = 0
self.reset_processing_timestamps()
def __init__(
self,
db: DBHandler,
user_directory: Path,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(
database=db,
user_directory=user_directory,
create_csv=create_csv,
)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = 0
self.currently_processing_timestamp = -1
self.first_processed_timestamp = -1
def __init__(
self,
price_historian,
profit_currency,
user_directory,
create_csv,
ignored_assets,
include_crypto2crypto,
taxfree_after_period,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
# Customizable Options
self.ignored_assets = ignored_assets
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
class Accountant():
def __init__(
self,
db: DBHandler,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory, create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = Timestamp(-1)
self.currently_processing_timestamp = Timestamp(-1)
def __del__(self) -> None:
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def _customize(self, settings: DBSettings) -> None:
"""Customize parameters after pulling DBSettings"""
if settings.include_crypto2crypto is not None:
self.events.include_crypto2crypto = settings.include_crypto2crypto
if settings.taxfree_after_period is not None:
given_taxfree_after_period: Optional[int] = settings.taxfree_after_period
if given_taxfree_after_period == -1:
# That means user requested to disable taxfree_after_period
given_taxfree_after_period = None
self.events.taxfree_after_period = given_taxfree_after_period
self.profit_currency = settings.main_currency
self.events.profit_currency = settings.main_currency
self.csvexporter.profit_currency = settings.main_currency
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
"""
Adds the given asset movement to the processed events
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
timestamp = movement.timestamp
if timestamp < self.start_ts:
return
if movement.asset.identifier == 'KFEE':
# There is no reason to process deposits of KFEE for kraken as it has only value
# internal to kraken and KFEE has no value and will error at cryptocompare price query
return
fee_rate = self.events.get_rate_in_profit_currency(movement.fee_asset, timestamp)
cost = movement.fee * fee_rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=movement.category,
asset=movement.asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=movement.location,
)
self.csvexporter.add_asset_movement(
exchange=movement.location,
category=movement.category,
asset=movement.asset,
fee=movement.fee,
rate=fee_rate,
timestamp=timestamp,
)
def account_for_gas_costs(
self,
transaction: EthereumTransaction,
include_gas_costs: bool,
) -> None:
"""
Accounts for the gas costs of the given ethereum transaction
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if not include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.events.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10 ** 18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.tx_hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade, loan_settlement: bool) -> None:
"""
Adds the given trade to the sell events
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.events.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Union[Trade, MarginPosition]],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
# Used only in the "avoid zerorpc remote lost after 10ms problem"
self.last_sleep_ts = 0
# Ask the DB for the settings once at the start of processing so we got the
# same settings through the entire task
db_settings = self.db.get_settings()
self._customize(db_settings)
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(
key=lambda action: action_get_timestamp(action),
)
# The first ts is the ts of the first action we have in history or 0 for empty history
first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(actions[0])
self.currently_processing_timestamp = first_ts
self.started_processing_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count, db_settings)
except PriceQueryUnknownFromAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}',
)
continue
except RemoteError as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to reach an external '
f'service at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to reach an external service at that time: {str(e)}',
)
continue
if not should_continue:
break
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (
self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs
)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit': str(self.events.loan_profit),
'margin_positions_profit_loss': str(self.events.margin_positions_profit_loss),
'settlement_losses': str(self.events.settlement_losses),
'ethereum_transaction_gas_costs': str(self.eth_transactions_gas_costs),
'asset_movement_fees': str(self.asset_movement_fees),
'general_trade_profit_loss': str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss': str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss': str(total_taxable_pl),
'total_profit_loss': str(
self.events.general_trade_profit_loss +
sum_other_actions,
),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
db_settings: DBSettings,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.',
)
return True, prev_time, count
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant():
def __init__(
self,
db: DBHandler,
user_directory: Path,
msg_aggregator: MessagesAggregator,
create_csv: bool,
) -> None:
self.db = db
profit_currency = db.get_main_currency()
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(
database=db,
user_directory=user_directory,
create_csv=create_csv,
)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = 0
self.currently_processing_timestamp = -1
self.first_processed_timestamp = -1
def __del__(self) -> None:
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def _customize(self, settings: DBSettings) -> None:
"""Customize parameters after pulling DBSettings"""
if settings.include_crypto2crypto is not None:
self.events.include_crypto2crypto = settings.include_crypto2crypto
if settings.taxfree_after_period is not None:
given_taxfree_after_period: Optional[
int] = settings.taxfree_after_period
if given_taxfree_after_period == -1:
# That means user requested to disable taxfree_after_period
given_taxfree_after_period = None
self.events.taxfree_after_period = given_taxfree_after_period
self.profit_currency = settings.main_currency
self.events.profit_currency = settings.main_currency
self.csvexporter.profit_currency = settings.main_currency
if settings.account_for_assets_movements is not None:
self.events.account_for_assets_movements = settings.account_for_assets_movements
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
"""Get the profit_currency rate of the fee of the given trade
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
fee_rate = PriceHistorian().query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return Fee(fee_rate * trade.fee)
def add_asset_movement_to_events(self, movement: AssetMovement) -> None:
"""
Adds the given asset movement to the processed events
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
timestamp = movement.timestamp
if timestamp < self.start_ts:
return
if movement.asset.identifier == 'KFEE' or not self.events.account_for_assets_movements:
# There is no reason to process deposits of KFEE for kraken as it has only value
# internal to kraken and KFEE has no value and will error at cryptocompare price query
return
fee_rate = self.events.get_rate_in_profit_currency(
movement.fee_asset, timestamp)
cost = movement.fee * fee_rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=movement.category,
asset=movement.asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=movement.location,
)
self.csvexporter.add_asset_movement(
exchange=movement.location,
category=movement.category,
asset=movement.asset,
fee=movement.fee,
rate=fee_rate,
timestamp=timestamp,
)
def account_for_gas_costs(
self,
transaction: EthereumTransaction,
include_gas_costs: bool,
) -> None:
"""
Accounts for the gas costs of the given ethereum transaction
May raise:
- PriceQueryUnsupportedAsset if from/to assets are missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
if not include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.events.get_rate_in_profit_currency(A_ETH,
transaction.timestamp)
eth_burned_as_gas = FVal(transaction.gas_used * gas_price) / FVal(10**
18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.tx_hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
"""
Adds the given trade to the sell events
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from all price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.events.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
location=trade.location,
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
location=trade.location,
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Union[Trade, MarginPosition, AMMTrade]],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
defi_events: List[DefiEvent],
ledger_actions: List[LedgerAction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = 2000000000
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset()
# Ask the DB for the settings once at the start of processing so we got the
# same settings through the entire task
db_settings = self.db.get_settings()
self._customize(db_settings)
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
if len(defi_events) != 0:
actions.extend(defi_events)
if len(ledger_actions) != 0:
actions.extend(ledger_actions)
actions.sort(key=action_get_timestamp)
# The first ts is the ts of the first action we have in history or 0 for empty history
first_ts = Timestamp(0) if len(actions) == 0 else action_get_timestamp(
actions[0])
self.currently_processing_timestamp = first_ts
self.first_processed_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
ignored_actionids_mapping = self.db.get_ignored_action_ids(
action_type=None)
for action in actions:
try:
(
should_continue,
prev_time,
) = self._process_action(
action=action,
start_ts=start_ts,
end_ts=end_ts,
prev_time=prev_time,
db_settings=db_settings,
ignored_actionids_mapping=ignored_actionids_mapping,
)
except PriceQueryUnsupportedAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details', )
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}', )
continue
except RemoteError as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f'{self.csvexporter.timestamp_to_date(ts)} '
f'during history processing due to inability to reach an external '
f'service at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to reach an external service at that time: {str(e)}',
)
continue
if not should_continue:
break
if count % 500 == 0:
# This loop can take a very long time depending on the amount of actions
# to process. We need to yield to other greenlets or else calls to the
# API may time out
gevent.sleep(0.5)
count += 1
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.defi_profit_loss +
self.events.ledger_actions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'ledger_actions_profit_loss':
str(self.events.ledger_actions_profit_loss),
'defi_profit_loss':
str(self.events.defi_profit_loss),
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
@staticmethod
def _should_ignore_action(
action: TaxableAction,
action_type: str,
ignored_actionids_mapping: Dict[ActionType, List[str]],
) -> Tuple[bool, Any]:
# TODO: These ifs/mappings of action type str to the enum
# are only due to mix of new and old code. They should be removed and only
# the enum should be used everywhere eventually
should_ignore = False
identifier: Optional[Any] = None
if action_type == 'trade':
trade = cast(Trade, action)
identifier = trade.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.TRADE, [])
elif action_type == 'asset_movement':
movement = cast(AssetMovement, action)
identifier = movement.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.ASSET_MOVEMENT,
[],
)
elif action_type == 'ethereum_transaction':
tx = cast(EthereumTransaction, action)
identifier = tx.identifier
should_ignore = tx.identifier in ignored_actionids_mapping.get(
ActionType.ETHEREUM_TX,
[],
)
elif action_type == 'ledger_action':
ledger_action = cast(LedgerAction, action)
identifier = ledger_action.identifier
should_ignore = identifier in ignored_actionids_mapping.get(
ActionType.LEDGER_ACTION,
[],
)
return should_ignore, identifier
def _process_action(
self,
action: TaxableAction,
start_ts: Timestamp,
end_ts: Timestamp,
prev_time: Timestamp,
db_settings: DBSettings,
ignored_actionids_mapping: Dict[ActionType, List[str]],
) -> Tuple[bool, Timestamp]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if not db_settings.calculate_past_cost_basis and timestamp < start_ts:
# ignore older actions than start_ts if we don't want past cost basis
return True, prev_time
if timestamp > end_ts:
# reached the end of the time period for the report
return False, prev_time
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except DeserializationError:
self.msg_aggregator.add_error(
'At history processing found trade with non string asset type. '
'Ignoring the trade.', )
return True, prev_time
if isinstance(asset1, UnknownEthereumToken) or isinstance(
asset2, UnknownEthereumToken): # type: ignore # noqa: E501
# TODO: Typing needs fixing here # type: ignore
log.debug( # type: ignore
'Ignoring action with unknown token',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
should_ignore, identifier = self._should_ignore_action(
action=action,
action_type=action_type,
ignored_actionids_mapping=ignored_actionids_mapping,
)
if should_ignore:
log.info(
f'Ignoring {action_type} action with identifier {identifier} '
f'at {timestamp} since the user asked to ignore it', )
return True, prev_time
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
location=action.location,
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time
if action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time
if action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time
if action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time
if action_type == 'defi_event':
action = cast(DefiEvent, action)
self.events.add_defi_event(action)
return True, prev_time
if action_type == 'ledger_action':
action = cast(LedgerAction, action)
self.events.add_ledger_action(action)
return True, prev_time
# else if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
location=trade.location,
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
location=trade.location,
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant(object):
def __init__(
self,
price_historian: PriceHistorian,
profit_currency: FiatAsset,
user_directory: FilePath,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __del__(self):
del self.events
del self.csvexporter
del self.price_historian
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
include_c2c = self.events.include_crypto2crypto
taxfree_after_period = self.events.taxfree_after_period
if 'include_crypto2crypto' in settings:
include_c2c = settings['include_crypto2crypto']
if not isinstance(include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
if 'taxfree_after_period' in settings:
taxfree_after_period = settings['taxfree_after_period']
if taxfree_after_period is not None:
if not isinstance(taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
taxfree_after_period = taxfree_after_period * 86400
settings['taxfree_after_period'] = taxfree_after_period
self.events.include_crypto2crypto = include_c2c
self.events.taxfree_after_period = taxfree_after_period
return True, ''
def set_main_currency(self, currency: FiatAsset) -> None:
if currency not in FIAT_CURRENCIES:
raise ValueError(
'Attempted to set unsupported "{}" as main currency.'.format(
currency), )
self.profit_currency = currency
self.events.profit_currency = currency
def query_historical_price(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> FVal:
price = self.price_historian.query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = self.query_historical_price(
asset,
self.profit_currency,
timestamp,
)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
fee_rate = self.query_historical_price(
trade.fee_currency,
self.profit_currency,
trade.timestamp,
)
return fee_rate * trade.fee
def add_asset_movement_to_events(
self,
category: str,
asset: Asset,
amount: FVal,
timestamp: Timestamp,
exchange: str,
fee: Fee,
) -> None:
if timestamp < self.start_ts:
return
rate = self.get_rate_in_profit_currency(asset, timestamp)
cost = fee * rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=category,
asset=asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=exchange,
)
if category == 'withdrawal':
assert fee != 0, 'So far all exchanges charge you for withdrawing'
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
if not self.include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency(S_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
selling_asset = trade_get_other_pair(trade, trade.cost_currency)
selling_asset_rate = self.get_rate_in_profit_currency(
trade.cost_currency,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=trade.cost_currency,
receiving_amount=trade.cost,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Trade],
margin_history: List[Trade],
loan_history: Dict,
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(margin_history) != 0:
actions.extend(margin_history)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action), )
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count)
except PriceQueryUnknownFromAsset as e:
log.error(
f'Skipping trade during history processing: {str(e)}')
continue
if not should_continue:
break
self.events.calculate_asset_details()
self.price_historian.inquirer.save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not"""
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client.
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely by async
count += 1
if count % 500 == 0:
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
action_type = action_get_type(action)
asset1, asset2 = action_get_assets(action)
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Dict, action)
self.events.add_loan_gain(
gained_asset=action['currency'],
lent_amount=action['amount_lent'],
gained_amount=action['earned'],
fee_in_asset=action['fee'],
open_time=action['open_time'],
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(
category=action.category,
asset=action.asset,
amount=action.amount,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee,
)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(
gain_loss_asset=action.pl_currency,
gain_loss_amount=action.profit_loss,
fee_in_asset=Fee(ZERO),
margin_notes=action.notes,
timestamp=action.close_time,
)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# if the cost is not equal to rate * amount then the data is somehow corrupt
if not trade.cost.is_close(trade.amount * trade.rate, max_diff="1e-4"):
raise CorruptData(
"Trade found with cost {} which is not equal to trade.amount"
"({}) * trade.rate({})".format(trade.cost, trade.amount,
trade.rate), )
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.type == 'buy':
self.events.add_buy_and_corresponding_sell(
bought_asset=trade_get_other_pair(trade, trade.cost_currency),
bought_amount=trade.amount,
paid_with_asset=trade.cost_currency,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.type == 'sell':
self.trade_add_to_sell_events(trade, False)
elif trade.type == 'settlement_sell':
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.type == 'settlement_buy':
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = S_BTC
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
selling_amount = trade.cost
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
raise ValueError('Unknown trade type "{}" encountered'.format(
trade.type))
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant(object):
def __init__(self, price_historian, profit_currency, user_directory,
create_csv, ignored_assets):
self.price_historian = price_historian
self.set_main_currency(profit_currency)
self.ignored_assets = ignored_assets
# If this flag is True when your asset is being forcefully sold as a
# loan/margin settlement then profit/loss is also calculated before the entire
# amount is taken as a loss
self.count_profit_for_settlements = False
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
def set_main_currency(self, currency):
if currency not in FIAT_CURRENCIES:
raise ValueError(
'Attempted to set unsupported "{}" as main currency.'.format(
currency))
self.profit_currency = currency
def query_historical_price(self, from_asset, to_asset, timestamp):
price = self.price_historian.query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset, timestamp):
if asset == self.profit_currency:
rate = 1
else:
rate = self.query_historical_price(asset, self.profit_currency,
timestamp)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def handle_prefork_acquisitions(self, bought_asset, bought_amount,
paid_with_asset, trade_rate, trade_fee,
fee_currency, timestamp):
# TODO: Should fee also be taken into account here?
if bought_asset == 'ETH' and timestamp < ETH_DAO_FORK_TS:
# Acquiring ETH before the DAO fork provides equal amount of ETC
self.add_buy_to_events('ETC',
bought_amount,
paid_with_asset,
trade_rate,
0,
fee_currency,
timestamp,
is_virtual=True)
if bought_asset == 'BTC' and timestamp < BTC_BCH_FORK_TS:
# Acquiring BTC before the BCH fork provides equal amount of BCH
self.add_buy_to_events('BCH',
bought_amount,
paid_with_asset,
trade_rate,
0,
fee_currency,
timestamp,
is_virtual=True)
def add_buy_to_events(self,
bought_asset,
bought_amount,
paid_with_asset,
trade_rate,
trade_fee,
fee_currency,
timestamp,
is_virtual=False):
paid_with_asset_rate = self.get_rate_in_profit_currency(
paid_with_asset, timestamp)
buy_rate = paid_with_asset_rate * trade_rate
fee_price_in_profit_currency = 0
if trade_fee != 0:
fee_price_in_profit_currency = self.query_historical_price(
fee_currency, self.profit_currency, timestamp)
self.handle_prefork_acquisitions(bought_asset=bought_asset,
bought_amount=bought_amount,
paid_with_asset=paid_with_asset,
trade_rate=trade_rate,
trade_fee=trade_fee,
fee_currency=fee_currency,
timestamp=timestamp)
if bought_asset not in self.events:
self.events[bought_asset] = Events(list(), list())
fee_cost = fee_price_in_profit_currency * trade_fee
gross_cost = bought_amount * buy_rate
cost = gross_cost + fee_cost
self.events[bought_asset].buys.append(
BuyEvent(amount=bought_amount,
timestamp=timestamp,
rate=buy_rate,
fee_rate=fee_cost / bought_amount,
cost=cost))
if logger.isEnabledFor(logging.DEBUG):
logger.debug(
'Buying {} "{}" for {} "{}" ({} "{}" per "{}" or {} "{}" per '
'"{}") at {}'.format(
bought_amount, bought_asset, bought_amount * trade_rate,
paid_with_asset, trade_rate, paid_with_asset, bought_asset,
buy_rate, self.profit_currency, bought_asset,
tsToDate(timestamp, formatstr='%d/%m/%Y %H:%M:%S')))
self.csvexporter.add_buy(
bought_asset=bought_asset,
rate=buy_rate,
fee_cost=fee_cost,
amount=bought_amount,
gross_cost=gross_cost,
cost=cost,
paid_with_asset=paid_with_asset,
paid_with_asset_rate=paid_with_asset_rate,
timestamp=timestamp,
is_virtual=is_virtual,
)
def add_loan_gain_to_events(self, gained_asset, gained_amount,
fee_in_asset, lent_amount, open_time,
close_time):
timestamp = close_time
rate = self.get_rate_in_profit_currency(gained_asset, timestamp)
if gained_asset not in self.events:
self.events[gained_asset] = Events(list(), list())
net_gain_amount = gained_amount - fee_in_asset
gain_in_profit_currency = net_gain_amount * rate
assert gain_in_profit_currency > 0, "Loan profit is negative. Should never happen"
self.events[gained_asset].buys.append(
BuyEvent(amount=net_gain_amount,
timestamp=timestamp,
rate=rate,
fee_rate=0,
cost=0))
# count profits if we are inside the query period
if timestamp >= self.query_start_ts:
self.loan_profit += gain_in_profit_currency
self.csvexporter.add_loan_profit(
gained_asset=gained_asset,
gained_amount=gained_amount,
gain_in_profit_currency=gain_in_profit_currency,
lent_amount=lent_amount,
open_time=open_time,
close_time=close_time,
)
def add_margin_positions_to_events(self, gained_asset, gained_amount,
fee_in_asset, margin_notes, timestamp):
rate = self.get_rate_in_profit_currency(gained_asset, timestamp)
if gained_asset not in self.events:
self.events[gained_asset] = Events(list(), list())
net_gain_amount = gained_amount - fee_in_asset
gain_in_profit_currency = net_gain_amount * rate
assert gain_in_profit_currency > 0, (
'Margin profit is negative. Should never happen for the hacky way I use em now'
)
self.events[gained_asset].buys.append(
BuyEvent(amount=net_gain_amount,
timestamp=timestamp,
rate=rate,
fee_rate=0,
cost=0))
# count profits if we are inside the query period
if timestamp >= self.query_start_ts:
self.margin_positions_profit += gain_in_profit_currency
self.csvexporter.add_margin_position(
margin_notes=margin_notes,
gained_asset=gained_asset,
net_gain_amount=net_gain_amount,
gain_in_profit_currency=gain_in_profit_currency,
timestamp=timestamp,
)
def add_asset_movement_to_events(self, category, asset, amount, timestamp,
exchange, fee):
rate = self.get_rate_in_profit_currency(asset, timestamp)
self.asset_movement_fees += fee * rate
if category == 'withdrawal':
assert fee != 0, "So far all exchanges charge you for withdrawing"
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction):
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency('ETH', transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
self.eth_transactions_gas_costs += eth_burned_as_gas * rate
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def add_buy_to_events_and_corresponding_sell(self, bought_asset,
bought_amount,
paid_with_asset, trade_rate,
trade_fee, fee_currency,
timestamp):
if logger.isEnabledFor(logging.DEBUG):
logger.debug('\nBUY EVENT:')
self.add_buy_to_events(
bought_asset=bought_asset,
bought_amount=bought_amount,
paid_with_asset=paid_with_asset,
trade_rate=trade_rate,
trade_fee=trade_fee,
fee_currency=fee_currency,
timestamp=timestamp,
is_virtual=False,
)
if paid_with_asset not in FIAT_CURRENCIES:
# then you are also selling some other asset to buy the bought asset
try:
bought_asset_rate_in_profit_currency = self.get_rate_in_profit_currency(
bought_asset, timestamp)
except (NoPriceForGivenTimestamp, PriceQueryUnknownFromAsset):
bought_asset_rate_in_profit_currency = -1
if bought_asset_rate_in_profit_currency != -1:
# The asset bought does not have a price yet
# Can happen for Token sales, presales e.t.c.
with_bought_asset_gain = bought_asset_rate_in_profit_currency * bought_amount
receiving_asset = bought_asset
receiving_amount = bought_amount
rate_in_profit_currency = bought_asset_rate_in_profit_currency / trade_rate
gain_in_profit_currency = with_bought_asset_gain
sold_amount = trade_rate * bought_amount
sold_asset_rate_in_profit_currency = self.get_rate_in_profit_currency(
paid_with_asset, timestamp)
with_sold_asset_gain = sold_asset_rate_in_profit_currency * sold_amount
# Consider as value of the sell what would give the least profit
if (bought_asset_rate_in_profit_currency == -1
or with_sold_asset_gain < with_bought_asset_gain):
receiving_asset = self.profit_currency
receiving_amount = with_sold_asset_gain
trade_rate = sold_asset_rate_in_profit_currency
rate_in_profit_currency = sold_asset_rate_in_profit_currency
gain_in_profit_currency = with_sold_asset_gain
# add the fee
fee_rate = self.get_rate_in_profit_currency(
fee_currency, timestamp)
fee_in_profit_currency = fee_rate * trade_fee
self.add_sell_to_events(
selling_asset=paid_with_asset,
selling_amount=sold_amount,
receiving_asset=receiving_asset,
receiving_amount=receiving_amount,
trade_rate=trade_rate,
rate_in_profit_currency=rate_in_profit_currency,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
timestamp=timestamp,
is_virtual=True,
)
def search_buys_calculate_profit(self, selling_amount, selling_asset,
timestamp):
"""
When selling `selling_amount` of `selling_asset` at `timestamp` this function
calculates using the first-in-first-out rule the corresponding buy/s from
which to do profit calculation. Also applies the one year rule after which
a sell is not taxable in Germany.
Returns a tuple of 3 values:
- `taxable_amount`: The amount out of `selling_amount` that is taxable,
calculated from the 1 year rule.
- `taxable_bought_cost`: How much it cost in `profit_currency` to buy
the `taxable_amount`
- `taxfree_bought_cost`: How much it cost in `profit_currency` to buy
the taxfree_amount (selling_amount - taxable_amount)
"""
remaining_sold_amount = selling_amount
stop_index = -1
taxfree_bought_cost = 0
taxable_bought_cost = 0
taxable_amount = 0
taxfree_amount = 0
debug_enabled = logger.isEnabledFor(logging.DEBUG)
for idx, buy_event in enumerate(self.events[selling_asset].buys):
sell_after_year = buy_event.timestamp + YEAR_IN_SECONDS < timestamp
if remaining_sold_amount < buy_event.amount:
stop_index = idx
buying_cost = remaining_sold_amount.fma(
buy_event.rate,
-(buy_event.fee_rate * remaining_sold_amount))
if sell_after_year:
taxfree_amount += remaining_sold_amount
taxfree_bought_cost += buying_cost
else:
taxable_amount += remaining_sold_amount
taxable_bought_cost += buying_cost
remaining_amount_from_last_buy = buy_event.amount - remaining_sold_amount
if debug_enabled:
logger.debug(
'[{}] Using up {}/{} "{}" from the buy for {} "{}" per "{}" at {}'
.format(
'TAX-FREE' if sell_after_year else 'TAXABLE',
remaining_sold_amount, buy_event.amount,
selling_asset, buy_event.rate,
self.profit_currency, selling_asset,
tsToDate(buy_event.timestamp,
formatstr='%d/%m/%Y %H:%M:%S')))
# stop iterating since we found all buys to satisfy this sell
break
else:
remaining_sold_amount -= buy_event.amount
if sell_after_year:
taxfree_amount += buy_event.amount
taxfree_bought_cost += buy_event.cost
else:
taxable_amount += buy_event.amount
taxable_bought_cost += buy_event.cost
if debug_enabled:
logger.debug(
'[{}] Using up the entire buy of {} "{}" for {} "{}" per {} at {}'
.format(
'TAX-FREE' if sell_after_year else 'TAXABLE',
buy_event.amount, selling_asset, buy_event.rate,
self.profit_currency, selling_asset,
tsToDate(buy_event.timestamp,
formatstr='%d/%m/%Y %H:%M:%S')))
if stop_index == -1:
logger.critical(
'No documented buy found for "{}" before {}'.format(
selling_asset,
tsToDate(timestamp, formatstr='%d/%m/%Y %H:%M:%S')))
# That means we had no documented buy for that asset. This is not good
# because we can't prove a corresponding buy and as such we are burdened
# calculating the entire sell as profit which needs to be taxed
return selling_amount, 0, 0
# Otherwise, delete all the used up buys from the list
del self.events[selling_asset].buys[:stop_index]
# and modify the amount of the buy where we stopped
self.events[selling_asset].buys[0] = self.events[selling_asset].buys[
0]._replace(amount=remaining_amount_from_last_buy)
return taxable_amount, taxable_bought_cost, taxfree_bought_cost
def add_sell_to_events(self,
selling_asset,
selling_amount,
receiving_asset,
receiving_amount,
gain_in_profit_currency,
total_fee_in_profit_currency,
trade_rate,
rate_in_profit_currency,
timestamp,
loan_settlement=False,
is_virtual=False):
if selling_asset not in self.events:
self.events[selling_asset] = Events(list(), list())
self.events[selling_asset].sells.append(
SellEvent(
amount=selling_amount,
timestamp=timestamp,
rate=rate_in_profit_currency,
fee_rate=total_fee_in_profit_currency / selling_amount,
gain=gain_in_profit_currency,
))
debug_enabled = logger.isEnabledFor(logging.DEBUG)
if debug_enabled:
if loan_settlement:
logger.debug(
'Loan Settlement Selling {} of "{}" for {} "{}" at {}'.
format(selling_amount, selling_asset,
gain_in_profit_currency, self.profit_currency,
tsToDate(timestamp, formatstr='%d/%m/%Y %H:%M:%S')))
else:
logger.debug(
'Selling {} of "{}" for {} "{}" ({} "{}" per "{}" or {} "{}" '
'per "{}") for a gain of {} "{}" and a fee of {} "{} at {}'
.format(selling_amount, selling_asset, receiving_amount,
receiving_asset, trade_rate, receiving_asset,
selling_asset, rate_in_profit_currency,
self.profit_currency, selling_asset,
gain_in_profit_currency, self.profit_currency,
total_fee_in_profit_currency, self.profit_currency,
tsToDate(timestamp,
formatstr='%d/%m/%Y %H:%M:%S')))
# now search the buys for `paid_with_asset` and calculate profit/loss
(taxable_amount, taxable_bought_cost,
taxfree_bought_cost) = self.search_buys_calculate_profit(
selling_amount, selling_asset, timestamp)
general_profit_loss = 0
taxable_profit_loss = 0
# and then calculate profit/loss
if not loan_settlement or (loan_settlement
and self.count_profit_for_settlements):
taxable_gain = taxable_gain_for_sell(
taxable_amount=taxable_amount,
rate_in_profit_currency=rate_in_profit_currency,
total_fee_in_profit_currency=total_fee_in_profit_currency,
selling_amount=selling_amount,
)
general_profit_loss = gain_in_profit_currency - (
taxfree_bought_cost + taxable_bought_cost +
total_fee_in_profit_currency)
taxable_profit_loss = taxable_gain - taxable_bought_cost
# should never happen, should be stopped at the main loop
assert timestamp <= self.query_end_ts, (
"Trade time > query_end_ts found in adding to sell event")
# count profit/losses if we are inside the query period
if timestamp >= self.query_start_ts:
if loan_settlement:
# If it's a loan settlement we are charged both the fee and the gain
settlement_loss = gain_in_profit_currency + total_fee_in_profit_currency
self.settlement_losses += settlement_loss
if debug_enabled:
logger.debug("Loan Settlement Loss: {} {}".format(
settlement_loss, self.profit_currency))
elif debug_enabled:
logger.debug("Taxable P/L: {} {} General P/L: {} {}".format(
taxable_profit_loss,
self.profit_currency,
general_profit_loss,
self.profit_currency,
))
self.general_trade_profit_loss += general_profit_loss
self.taxable_trade_profit_loss += taxable_profit_loss
if loan_settlement:
self.csvexporter.add_loan_settlement(
asset=selling_asset,
amount=selling_amount,
rate_in_profit_currency=rate_in_profit_currency,
total_fee_in_profit_currency=total_fee_in_profit_currency,
timestamp=timestamp,
)
else:
self.csvexporter.add_sell(
selling_asset=selling_asset,
rate_in_profit_currency=rate_in_profit_currency,
total_fee_in_profit_currency=total_fee_in_profit_currency,
gain_in_profit_currency=gain_in_profit_currency,
selling_amount=selling_amount,
receiving_asset=receiving_asset,
receiving_amount=receiving_amount,
receiving_asset_rate_in_profit_currency=self.
get_rate_in_profit_currency(
receiving_asset,
timestamp,
),
taxable_amount=taxable_amount,
taxable_bought_cost=taxable_bought_cost,
timestamp=timestamp,
is_virtual=is_virtual,
)
def add_sell_to_events_and_corresponding_buy(
self, selling_asset, selling_amount, receiving_asset,
receiving_amount, gain_in_profit_currency,
total_fee_in_profit_currency, trade_rate, rate_in_profit_currency,
timestamp):
"""
Add and process a selling event to the events list
Args:
selling_asset (str): The ticker representation of the asset we sell.
selling_amount (FVal): The amount of `selling_asset` for sale.
receiving_asset (str): The ticker representation of the asset we receive
in exchange for `selling_asset`.
receiving_amount (FVal): The amount of `receiving_asset` we receive.
gain_in_profit_currency (FVal): This is the amount of `profit_currency` equivalent
we receive after doing this trade. Fees are not counted
in this.
total_fee_in_profit_currency (FVal): This is the amount of `profit_currency` equivalent
we pay in fees after doing this trade.
trade_rate (FVal): How much does 1 unit of `receiving_asset` cost in `selling_asset`
rate_in_profit_currency (FVal): The equivalent of `trade_rate` in `profit_currency`
timestamp (int): The timestamp for the trade
"""
if logger.isEnabledFor(logging.DEBUG):
logger.debug('\nSELL EVENT:')
self.add_sell_to_events(
selling_asset,
selling_amount,
receiving_asset,
receiving_amount,
gain_in_profit_currency,
total_fee_in_profit_currency,
trade_rate,
rate_in_profit_currency,
timestamp,
is_virtual=False,
)
if receiving_asset not in FIAT_CURRENCIES:
# then you are also buying some other asset through your sell
# TODO: Account for trade fees in the virtual buy too
self.add_buy_to_events(
bought_asset=receiving_asset,
bought_amount=receiving_amount,
paid_with_asset=selling_asset,
trade_rate=1 / trade_rate,
trade_fee=0,
fee_currency=receiving_amount, # does not matter
timestamp=timestamp,
is_virtual=True,
)
def save_events(self):
for asset, events in self.events.items():
pass
def calculate_asset_details(self):
""" Calculates what amount of all assets has been untouched for a year and
is hence tax-free and also the average buy price for each asset"""
self.details = dict()
now = ts_now()
for asset, events in self.events.items():
tax_free_amount_left = 0
amount_sum = 0
average = 0
for buy_event in events.buys:
if buy_event.timestamp + YEAR_IN_SECONDS < now:
tax_free_amount_left += buy_event.amount
amount_sum += buy_event.amount
average += buy_event.amount * buy_event.rate
if amount_sum == 0:
self.details[asset] = (0, 0)
else:
self.details[asset] = (tax_free_amount_left,
average / amount_sum)
return self.details
def trade_add_to_sell_events(self, trade, loan_settlement):
selling_asset = trade_get_other_pair(trade, trade.cost_currency)
selling_asset_rate = self.get_rate_in_profit_currency(
trade.cost_currency, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.add_sell_to_events_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=trade.cost_currency,
receiving_amount=trade.cost,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.add_sell_to_events(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False)
def process_history(self, start_ts, end_ts, trade_history, margin_history,
loan_history, asset_movements, eth_transactions):
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
"""
self.events = dict()
self.general_trade_profit_loss = FVal(0)
self.taxable_trade_profit_loss = FVal(0)
self.settlement_losses = FVal(0)
self.loan_profit = FVal(0)
self.margin_positions_profit = FVal(0)
self.last_gas_price = FVal("2000000000")
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.query_start_ts = start_ts
self.query_end_ts = end_ts
self.csvexporter.reset_csv_lists()
actions = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(margin_history) != 0:
actions.extend(margin_history)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action))
prev_time = 0
count = 0
for action in actions:
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client.
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely by async
count += 1
if count % 500 == 0:
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > self.query_end_ts:
break
action_type = action_get_type(action)
asset1, asset2 = action_get_assets(action)
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Ignoring {} with {} {}".format(
action_type, asset1, asset2))
continue
if action_type == 'loan':
self.add_loan_gain_to_events(
gained_asset=action['currency'],
lent_amount=action['amount_lent'],
gained_amount=action['earned'],
fee_in_asset=action['fee'],
open_time=action['open_time'],
close_time=timestamp,
)
continue
elif action_type == 'asset_movement':
self.add_asset_movement_to_events(category=action.category,
asset=action.asset,
amount=action.amount,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee)
continue
elif action_type == 'margin_position':
self.add_margin_positions_to_events(
gained_asset='BTC',
gained_amount=action['btc_profit_loss'],
fee_in_asset=0,
margin_notes=action['notes'],
timestamp=timestamp)
continue
elif action_type == 'ethereum_transaction':
self.account_for_gas_costs(action)
continue
# if we get here it's a trade
trade = action
# if the cost is not equal to rate * amount then the data is somehow corrupt
if not trade.cost.is_close(trade.amount * trade.rate,
max_diff="1e-5"):
raise CorruptData(
"Trade found with cost {} which is not equal to trade.amount"
"({}) * trade.rate({})".format(trade.cost, trade.amount,
trade.rate))
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.type == 'buy':
self.add_buy_to_events_and_corresponding_sell(
bought_asset=trade_get_other_pair(trade,
trade.cost_currency),
bought_amount=trade.amount,
paid_with_asset=trade.cost_currency,
trade_rate=trade.rate,
trade_fee=trade.fee,
fee_currency=trade.fee_currency,
timestamp=trade.timestamp)
elif trade.type == 'sell':
self.trade_add_to_sell_events(trade, False)
elif trade.type == 'settlement_sell':
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.type == 'settlement_buy':
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = 'BTC'
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
self.add_sell_to_events(
selling_asset=selling_asset,
selling_amount=trade.cost,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True)
else:
raise ValueError('Unknown trade type "{}" encountered'.format(
trade.type))
self.calculate_asset_details()
sum_other_actions = (self.margin_positions_profit + self.loan_profit -
self.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.loan_profit),
'margin_positions_profit':
str(self.margin_positions_profit),
'settlement_losses':
str(self.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.general_trade_profit_loss + sum_other_actions),
},
'all_events': self.csvexporter.all_events,
}
class Accountant():
def __init__(
self,
profit_currency: Asset,
user_directory: FilePath,
msg_aggregator: MessagesAggregator,
create_csv: bool,
ignored_assets: List[Asset],
include_crypto2crypto: bool,
taxfree_after_period: int,
include_gas_costs: bool,
):
self.msg_aggregator = msg_aggregator
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(self.csvexporter, profit_currency)
self.set_main_currency(profit_currency.identifier)
self.asset_movement_fees = FVal(0)
self.last_gas_price = FVal(0)
self.started_processing_timestamp = -1
self.currently_processing_timestamp = -1
# Customizable Options
self.ignored_assets = ignored_assets
self.include_gas_costs = include_gas_costs
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
def __del__(self):
del self.events
del self.csvexporter
@property
def general_trade_pl(self) -> FVal:
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self) -> FVal:
return self.events.taxable_trade_profit_loss
def customize(self, settings: Dict[str, Any]) -> Tuple[bool, str]:
if 'include_crypto2crypto' in settings:
given_include_c2c = settings['include_crypto2crypto']
if not isinstance(given_include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
self.events.include_crypto2crypto = given_include_c2c
if 'taxfree_after_period' in settings:
given_taxfree_after_period = settings['taxfree_after_period']
if given_taxfree_after_period is not None:
if not isinstance(given_taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if given_taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
given_taxfree_after_period = given_taxfree_after_period * 86400
settings['taxfree_after_period'] = given_taxfree_after_period
self.events.taxfree_after_period = given_taxfree_after_period
return True, ''
def set_main_currency(self, given_currency: str) -> None:
currency = Asset(given_currency)
msg = 'main currency checks should have happened at rotkehlchen.set_settings()'
assert currency.is_fiat(), msg
self.profit_currency = currency
self.events.profit_currency = currency
@staticmethod
def query_historical_price(
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
) -> FVal:
price = PriceHistorian().query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset: Asset,
timestamp: Timestamp) -> FVal:
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = FVal(1)
else:
rate = self.query_historical_price(
asset,
self.profit_currency,
timestamp,
)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def get_fee_in_profit_currency(self, trade: Trade) -> Fee:
fee_rate = self.query_historical_price(
from_asset=trade.fee_currency,
to_asset=self.profit_currency,
timestamp=trade.timestamp,
)
return fee_rate * trade.fee
def add_asset_movement_to_events(
self,
category: str,
asset: Asset,
timestamp: Timestamp,
exchange: Exchange,
fee: Fee,
) -> None:
if timestamp < self.start_ts:
return
rate = self.get_rate_in_profit_currency(asset, timestamp)
cost = fee * rate
self.asset_movement_fees += cost
log.debug(
'Accounting for asset movement',
sensitive_log=True,
category=category,
asset=asset,
cost_in_profit_currency=cost,
timestamp=timestamp,
exchange_name=exchange,
)
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction: EthereumTransaction) -> None:
if not self.include_gas_costs:
return
if transaction.timestamp < self.start_ts:
return
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency(A_ETH, transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
cost = eth_burned_as_gas * rate
self.eth_transactions_gas_costs += cost
log.debug(
'Accounting for ethereum transaction gas cost',
sensitive_log=True,
gas_used=transaction.gas_used,
gas_price=gas_price,
timestamp=transaction.timestamp,
)
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade: Trade,
loan_settlement: bool) -> None:
selling_asset = trade.base_asset
receiving_asset = trade.quote_asset
receiving_asset_rate = self.get_rate_in_profit_currency(
receiving_asset,
trade.timestamp,
)
selling_rate = receiving_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=receiving_asset,
receiving_amount=trade.amount * trade.rate,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False,
)
def process_history(
self,
start_ts: Timestamp,
end_ts: Timestamp,
trade_history: List[Trade],
loan_history: List[Loan],
asset_movements: List[AssetMovement],
eth_transactions: List[EthereumTransaction],
) -> Dict[str, Any]:
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
start_ts here is the timestamp at which to start taking trades and other
taxable events into account. Not where processing starts from. Processing
always starts from the very first event we find in the history.
"""
log.info(
'Start of history processing',
start_ts=start_ts,
end_ts=end_ts,
)
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.start_ts = start_ts
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
# Used only in the "avoid zerorpc remote lost after 10ms problem"
self.last_sleep_ts = 0
actions: List[TaxableAction] = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action), )
# The first timestamp is the timestamp of the first action we have in history
first_ts = action_get_timestamp(actions[0])
self.currently_processing_timestamp = first_ts
self.started_processing_timestamp = first_ts
prev_time = Timestamp(0)
count = 0
for action in actions:
try:
(
should_continue,
prev_time,
count,
) = self.process_action(action, end_ts, prev_time, count)
except PriceQueryUnknownFromAsset as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to an asset unknown to '
f'cryptocompare being involved. Check logs for details', )
log.error(
f'Skipping action {str(action)} during history processing due to '
f'cryptocompare not supporting an involved asset: {str(e)}',
)
continue
except NoPriceForGivenTimestamp as e:
ts = action_get_timestamp(action)
self.msg_aggregator.add_error(
f'Skipping action at '
f' {timestamp_to_date(ts, formatstr="%d/%m/%Y, %H:%M:%S")} '
f'during history processing due to inability to find a price '
f'at that point in time: {str(e)}. Check the logs for more details',
)
log.error(
f'Skipping action {str(action)} during history processing due to '
f'inability to query a price at that time: {str(e)}', )
continue
if not should_continue:
break
self.events.calculate_asset_details()
Inquirer().save_historical_forex_data()
sum_other_actions = (self.events.margin_positions_profit_loss +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit_loss':
str(self.events.margin_positions_profit_loss),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss +
sum_other_actions, ),
},
'all_events': self.csvexporter.all_events,
}
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not"""
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client. (after 10s)
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely be async
now = ts_now()
if now - self.last_sleep_ts >= 7: # choose 7 seconds to be safe
self.last_sleep_ts = now
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.', )
return True, prev_time, count
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(
category=action.category,
asset=action.asset,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee,
)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(
gain_loss_asset=action.pl_currency,
gain_loss_amount=action.profit_loss,
fee_in_asset=Fee(ZERO),
margin_notes=action.notes,
timestamp=action.close_time,
)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
raise ValueError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
def get_calculated_asset_amount(self, asset: Asset) -> Optional[FVal]:
"""Get the amount of asset accounting has calculated we should have after
the history has been processed
"""
if asset not in self.events.events:
return None
amount = FVal(0)
for buy_event in self.events.events[asset].buys:
amount += buy_event.amount
return amount
class Accountant(object):
def __init__(
self,
price_historian,
profit_currency,
user_directory,
create_csv,
ignored_assets,
include_crypto2crypto,
taxfree_after_period,
):
self.price_historian = price_historian
self.csvexporter = CSVExporter(profit_currency, user_directory,
create_csv)
self.events = TaxableEvents(price_historian, self.csvexporter,
profit_currency)
self.set_main_currency(profit_currency)
# Customizable Options
self.ignored_assets = ignored_assets
self.events.include_crypto2crypto = include_crypto2crypto
self.events.taxfree_after_period = taxfree_after_period
@property
def general_trade_pl(self):
return self.events.general_trade_profit_loss
@property
def taxable_trade_pl(self):
return self.events.taxable_trade_profit_loss
def customize(self, settings):
include_c2c = self.events.include_crypto2crypto
taxfree_after_period = self.events.taxfree_after_period
if 'include_crypto2crypto' in settings:
include_c2c = settings['include_crypto2crypto']
if not isinstance(include_c2c, bool):
return False, 'Value for include_crypto2crypto must be boolean'
if 'taxfree_after_period' in settings:
taxfree_after_period = settings['taxfree_after_period']
if taxfree_after_period is not None:
if not isinstance(taxfree_after_period, int):
return False, 'Value for taxfree_after_period must be an integer'
if taxfree_after_period == 0:
return False, 'Value for taxfree_after_period can not be 0 days'
# turn to seconds
taxfree_after_period = taxfree_after_period * 86400
settings['taxfree_after_period'] = taxfree_after_period
self.events.include_crypto2crypto = include_c2c
self.events.taxfree_after_period = taxfree_after_period
return True, ''
def set_main_currency(self, currency):
if currency not in FIAT_CURRENCIES:
raise ValueError(
'Attempted to set unsupported "{}" as main currency.'.format(
currency))
self.profit_currency = currency
self.events.profit_currency = currency
def query_historical_price(self, from_asset, to_asset, timestamp):
price = self.price_historian.query_historical_price(
from_asset, to_asset, timestamp)
return price
def get_rate_in_profit_currency(self, asset, timestamp):
# TODO: Moved this to events.py too. Is it still needed here?
if asset == self.profit_currency:
rate = 1
else:
rate = self.query_historical_price(asset, self.profit_currency,
timestamp)
assert isinstance(rate,
(FVal, int)) # TODO Remove. Is temporary assert
return rate
def add_asset_movement_to_events(self, category, asset, amount, timestamp,
exchange, fee):
rate = self.get_rate_in_profit_currency(asset, timestamp)
self.asset_movement_fees += fee * rate
if category == 'withdrawal':
assert fee != 0, "So far all exchanges charge you for withdrawing"
self.csvexporter.add_asset_movement(
exchange=exchange,
category=category,
asset=asset,
fee=fee,
rate=rate,
timestamp=timestamp,
)
def account_for_gas_costs(self, transaction):
if transaction.gas_price == -1:
gas_price = self.last_gas_price
else:
gas_price = transaction.gas_price
self.last_gas_price = transaction.gas_price
rate = self.get_rate_in_profit_currency('ETH', transaction.timestamp)
eth_burned_as_gas = (transaction.gas_used * gas_price) / FVal(10**18)
self.eth_transactions_gas_costs += eth_burned_as_gas * rate
self.csvexporter.add_tx_gas_cost(
transaction_hash=transaction.hash,
eth_burned_as_gas=eth_burned_as_gas,
rate=rate,
timestamp=transaction.timestamp,
)
def trade_add_to_sell_events(self, trade, loan_settlement):
selling_asset = trade_get_other_pair(trade, trade.cost_currency)
selling_asset_rate = self.get_rate_in_profit_currency(
trade.cost_currency, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
if not loan_settlement:
self.events.add_sell_and_corresponding_buy(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=trade.cost_currency,
receiving_amount=trade.cost,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
)
else:
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True,
is_virtual=False)
def process_history(self, start_ts, end_ts, trade_history, margin_history,
loan_history, asset_movements, eth_transactions):
"""Processes the entire history of cryptoworld actions in order to determine
the price and time at which every asset was obtained and also
the general and taxable profit/loss.
"""
self.events.reset(start_ts, end_ts)
self.last_gas_price = FVal("2000000000")
self.eth_transactions_gas_costs = FVal(0)
self.asset_movement_fees = FVal(0)
self.csvexporter.reset_csv_lists()
actions = list(trade_history)
# If we got loans, we need to interleave them with the full history and re-sort
if len(loan_history) != 0:
actions.extend(loan_history)
if len(asset_movements) != 0:
actions.extend(asset_movements)
if len(margin_history) != 0:
actions.extend(margin_history)
if len(eth_transactions) != 0:
actions.extend(eth_transactions)
actions.sort(key=lambda action: action_get_timestamp(action))
prev_time = 0
count = 0
for action in actions:
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client.
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely by async
count += 1
if count % 500 == 0:
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
break
action_type = action_get_type(action)
asset1, asset2 = action_get_assets(action)
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
if logger.isEnabledFor(logging.DEBUG):
logger.debug("Ignoring {} with {} {}".format(
action_type, asset1, asset2))
continue
if action_type == 'loan':
self.events.add_loan_gain(
gained_asset=action['currency'],
lent_amount=action['amount_lent'],
gained_amount=action['earned'],
fee_in_asset=action['fee'],
open_time=action['open_time'],
close_time=timestamp,
)
continue
elif action_type == 'asset_movement':
self.add_asset_movement_to_events(category=action.category,
asset=action.asset,
amount=action.amount,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee)
continue
elif action_type == 'margin_position':
self.events.add_margin_position(
gained_asset='BTC',
gained_amount=action['btc_profit_loss'],
fee_in_asset=0,
margin_notes=action['notes'],
timestamp=timestamp)
continue
elif action_type == 'ethereum_transaction':
self.account_for_gas_costs(action)
continue
# if we get here it's a trade
trade = action
# if the cost is not equal to rate * amount then the data is somehow corrupt
if not trade.cost.is_close(trade.amount * trade.rate,
max_diff="1e-5"):
raise CorruptData(
"Trade found with cost {} which is not equal to trade.amount"
"({}) * trade.rate({})".format(trade.cost, trade.amount,
trade.rate))
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.type == 'buy':
self.events.add_buy_and_corresponding_sell(
bought_asset=trade_get_other_pair(trade,
trade.cost_currency),
bought_amount=trade.amount,
paid_with_asset=trade.cost_currency,
trade_rate=trade.rate,
trade_fee=trade.fee,
fee_currency=trade.fee_currency,
timestamp=trade.timestamp)
elif trade.type == 'sell':
self.trade_add_to_sell_events(trade, False)
elif trade.type == 'settlement_sell':
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.type == 'settlement_buy':
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = 'BTC'
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset, trade.timestamp)
selling_rate = selling_asset_rate * trade.rate
fee_rate = self.query_historical_price(trade.fee_currency,
self.profit_currency,
trade.timestamp)
total_sell_fee_cost = fee_rate * trade.fee
gain_in_profit_currency = selling_rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=trade.cost,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=total_sell_fee_cost,
trade_rate=trade.rate,
rate_in_profit_currency=selling_rate,
timestamp=trade.timestamp,
loan_settlement=True)
else:
raise ValueError('Unknown trade type "{}" encountered'.format(
trade.type))
self.events.calculate_asset_details()
sum_other_actions = (self.events.margin_positions_profit +
self.events.loan_profit -
self.events.settlement_losses -
self.asset_movement_fees -
self.eth_transactions_gas_costs)
total_taxable_pl = self.events.taxable_trade_profit_loss + sum_other_actions
return {
'overview': {
'loan_profit':
str(self.events.loan_profit),
'margin_positions_profit':
str(self.events.margin_positions_profit),
'settlement_losses':
str(self.events.settlement_losses),
'ethereum_transaction_gas_costs':
str(self.eth_transactions_gas_costs),
'asset_movement_fees':
str(self.asset_movement_fees),
'general_trade_profit_loss':
str(self.events.general_trade_profit_loss),
'taxable_trade_profit_loss':
str(self.events.taxable_trade_profit_loss),
'total_taxable_profit_loss':
str(total_taxable_pl),
'total_profit_loss':
str(self.events.general_trade_profit_loss + sum_other_actions),
},
'all_events': self.csvexporter.all_events,
}
