def _process_action(
self,
action: TaxableAction,
start_ts: Timestamp,
end_ts: Timestamp,
prev_time: Timestamp,
db_settings: DBSettings,
ignored_actionids_mapping: Dict[ActionType, List[str]],
) -> Tuple[bool, Timestamp]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnsupportedAsset if from/to asset is missing from price oracles
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from the price oracle
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if not db_settings.calculate_past_cost_basis and timestamp < start_ts:
# ignore older actions than start_ts if we don't want past cost basis
return True, prev_time
if timestamp > end_ts:
# reached the end of the time period for the report
return False, prev_time
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time
except DeserializationError:
self.msg_aggregator.add_error(
'At history processing found trade with non string asset type. '
'Ignoring the trade.', )
return True, prev_time
if isinstance(asset1, UnknownEthereumToken) or isinstance(
asset2, UnknownEthereumToken): # type: ignore # noqa: E501
# TODO: Typing needs fixing here # type: ignore
log.debug( # type: ignore
'Ignoring action with unknown token',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time
should_ignore, identifier = self._should_ignore_action(
action=action,
action_type=action_type,
ignored_actionids_mapping=ignored_actionids_mapping,
)
if should_ignore:
log.info(
f'Ignoring {action_type} action with identifier {identifier} '
f'at {timestamp} since the user asked to ignore it', )
return True, prev_time
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
location=action.location,
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time
if action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time
if action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time
if action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time
if action_type == 'defi_event':
action = cast(DefiEvent, action)
self.events.add_defi_event(action)
return True, prev_time
if action_type == 'ledger_action':
action = cast(LedgerAction, action)
self.events.add_ledger_action(action)
return True, prev_time
# else if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
location=trade.location,
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
location=trade.location,
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
db_settings: DBSettings,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not
May raise:
- PriceQueryUnknownFromAsset if the from asset is known to miss from cryptocompare
- NoPriceForGivenTimestamp if we can't find a price for the asset in the given
timestamp from cryptocompare
- RemoteError if there is a problem reaching the price oracle server
or with reading the response returned by the server
"""
ignored_assets = self.db.get_ignored_assets()
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.',
)
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.',
)
return True, prev_time, count
if asset1 in ignored_assets or asset2 in ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(action)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(margin=action)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action, db_settings.include_gas_costs)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.events.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
# Should never happen
raise AssertionError(f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
def process_action(
self,
action: TaxableAction,
end_ts: Timestamp,
prev_time: Timestamp,
count: int,
) -> Tuple[bool, Timestamp, int]:
"""Processes each individual action and returns whether we should continue
looping through the rest of the actions or not"""
# Hack to periodically yield back to the gevent IO loop to avoid getting
# the losing remote after hearbeat error for the zerorpc client. (after 10s)
# https://github.com/0rpc/zerorpc-python/issues/37
# TODO: Find better way to do this. Perhaps enforce this only if method
# is a synced call, and if async don't do this yielding. In any case
# this calculation should definitely be async
now = ts_now()
if now - self.last_sleep_ts >= 7: # choose 7 seconds to be safe
self.last_sleep_ts = now
gevent.sleep(0.01) # context switch
# Assert we are sorted in ascending time order.
timestamp = action_get_timestamp(action)
assert timestamp >= prev_time, (
"During history processing the trades/loans are not in ascending order"
)
prev_time = timestamp
if timestamp > end_ts:
return False, prev_time, count
self.currently_processing_timestamp = timestamp
action_type = action_get_type(action)
try:
asset1, asset2 = action_get_assets(action)
except UnknownAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unknown asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except UnsupportedAsset as e:
self.msg_aggregator.add_warning(
f'At history processing found trade with unsupported asset {e.asset_name}. '
f'Ignoring the trade.', )
return True, prev_time, count
except DeserializationError:
self.msg_aggregator.add_error(
f'At history processing found trade with non string asset type. '
f'Ignoring the trade.', )
return True, prev_time, count
if asset1 in self.ignored_assets or asset2 in self.ignored_assets:
log.debug(
'Ignoring action with ignored asset',
action_type=action_type,
asset1=asset1,
asset2=asset2,
)
return True, prev_time, count
if action_type == 'loan':
action = cast(Loan, action)
self.events.add_loan_gain(
gained_asset=action.currency,
lent_amount=action.amount_lent,
gained_amount=action.earned,
fee_in_asset=action.fee,
open_time=action.open_time,
close_time=timestamp,
)
return True, prev_time, count
elif action_type == 'asset_movement':
action = cast(AssetMovement, action)
self.add_asset_movement_to_events(
category=action.category,
asset=action.asset,
timestamp=action.timestamp,
exchange=action.exchange,
fee=action.fee,
)
return True, prev_time, count
elif action_type == 'margin_position':
action = cast(MarginPosition, action)
self.events.add_margin_position(
gain_loss_asset=action.pl_currency,
gain_loss_amount=action.profit_loss,
fee_in_asset=Fee(ZERO),
margin_notes=action.notes,
timestamp=action.close_time,
)
return True, prev_time, count
elif action_type == 'ethereum_transaction':
action = cast(EthereumTransaction, action)
self.account_for_gas_costs(action)
return True, prev_time, count
# if we get here it's a trade
trade = cast(Trade, action)
# When you buy, you buy with the cost_currency and receive the other one
# When you sell, you sell the amount in non-cost_currency and receive
# costs in cost_currency
if trade.trade_type == TradeType.BUY:
self.events.add_buy_and_corresponding_sell(
bought_asset=trade.base_asset,
bought_amount=trade.amount,
paid_with_asset=trade.quote_asset,
trade_rate=trade.rate,
fee_in_profit_currency=self.get_fee_in_profit_currency(trade),
fee_currency=trade.fee_currency,
timestamp=trade.timestamp,
)
elif trade.trade_type == TradeType.SELL:
self.trade_add_to_sell_events(trade, False)
elif trade.trade_type == TradeType.SETTLEMENT_SELL:
# in poloniex settlements sell some asset to get BTC to repay a loan
self.trade_add_to_sell_events(trade, True)
elif trade.trade_type == TradeType.SETTLEMENT_BUY:
# in poloniex settlements you buy some asset with BTC to repay a loan
# so in essense you sell BTC to repay the loan
selling_asset = A_BTC
selling_asset_rate = self.get_rate_in_profit_currency(
selling_asset,
trade.timestamp,
)
selling_rate = selling_asset_rate * trade.rate
fee_in_profit_currency = self.get_fee_in_profit_currency(trade)
gain_in_profit_currency = selling_rate * trade.amount
# Since the original trade is a buy of some asset with BTC, then the
# when we invert the sell, the sold amount of BTC should be the cost
# (amount*rate) of the original buy
selling_amount = trade.rate * trade.amount
self.events.add_sell(
selling_asset=selling_asset,
selling_amount=selling_amount,
receiving_asset=None,
receiving_amount=None,
gain_in_profit_currency=gain_in_profit_currency,
total_fee_in_profit_currency=fee_in_profit_currency,
trade_rate=trade.rate,
rate_in_profit_currency=selling_asset_rate,
timestamp=trade.timestamp,
loan_settlement=True,
)
else:
raise ValueError(
f'Unknown trade type "{trade.trade_type}" encountered')
return True, prev_time, count
