def maybe_add_external_trades_to_history(
db: DBHandler,
start_ts: Timestamp,
end_ts: Timestamp,
history: List[Union[Trade, MarginPosition]],
msg_aggregator: MessagesAggregator,
) -> List[Union[Trade, MarginPosition]]:
"""
Queries the DB to get any external trades, adds them to the provided history and returns it.
If there is an unexpected error at the external trade deserialization an error is logged.
"""
serialized_external_trades = db.get_trades()
try:
external_trades = trades_from_dictlist(
given_trades=serialized_external_trades,
start_ts=start_ts,
end_ts=end_ts,
location='external trades',
msg_aggregator=msg_aggregator,
)
except (KeyError, DeserializationError):
msg_aggregator.add_error(
'External trades in the DB are in an unrecognized format')
return history
history.extend(external_trades)
# TODO: We also sort in one other place in this file and also in accountant.py
# Get rid of the unneeded cases?
history.sort(key=lambda trade: action_get_timestamp(trade))
return history
def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[MarginPosition] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[AssetMovement] = None,
eth_transaction_list: List[Dict] = None,
defi_events_list: List[DefiEvent] = None,
) -> Dict[str, Any]:
trade_history: Sequence[Union[Trade, MarginPosition]]
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=Timestamp(0),
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
# if present, append margin positions to trade history
if margin_list:
trade_history.extend(margin_list) # type: ignore
asset_movements = []
if asset_movements_list:
asset_movements = asset_movements_list
loan_history = []
if loans_list:
loan_history = process_polo_loans(
msg_aggregator=accountant.msg_aggregator,
data=loans_list,
start_ts=Timestamp(0),
end_ts=end_ts,
)
eth_transactions = []
if eth_transaction_list:
eth_transactions = eth_transaction_list
defi_events = []
if defi_events_list:
defi_events = defi_events_list
result = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
defi_events=defi_events,
)
return result
def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[MarginPosition] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[Dict] = None,
eth_transaction_list: List[Dict] = None,
) -> Dict[str, Any]:
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=0,
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
# if present, append margin positions to trade history
if margin_list:
trade_history.extend(margin_list)
asset_movements = list()
if asset_movements_list:
asset_movements = asset_movements_from_dictlist(
given_data=asset_movements_list,
start_ts=0,
end_ts=end_ts,
)
loan_history = list()
if loans_list:
loan_history = process_polo_loans(
msg_aggregator=accountant.msg_aggregator,
data=loans_list,
start_ts=0,
end_ts=end_ts,
)
eth_transactions = list()
if eth_transaction_list:
eth_transactions = transactions_from_dictlist(
given_transactions=eth_transaction_list,
start_ts=0,
end_ts=end_ts,
)
result = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result
def test_trades_from_dictlist(function_scope_messages_aggregator):
raw_trades = [raw_trade1, raw_trade2, raw_trade3]
trades = trades_from_dictlist(
given_trades=raw_trades,
start_ts=1516985747,
end_ts=1557985736,
location='test',
msg_aggregator=function_scope_messages_aggregator,
)
assert len(trades) == 1
assert isinstance(trades[0], Trade)
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets all the cached history data instead of querying all external sources
to create the history through create_history()
Can raise:
- HistoryCacheInvalid:
If any of the cache files are corrupt in any way, missing or
do not cover the given time range
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
if not os.path.isfile(historyfile_path):
raise HistoryCacheInvalid()
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except JSONDecodeError:
pass
if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Historical trades cache invalid')
try:
history_trades = trades_from_dictlist(
given_trades=history_json_data['data'],
start_ts=start_ts,
end_ts=end_ts,
location='historical trades',
msg_aggregator=self.msg_aggregator,
)
except (KeyError, DeserializationError):
raise HistoryCacheInvalid('Historical trades cache invalid')
history_trades = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history_trades,
msg_aggregator=self.msg_aggregator,
)
# Check the cache of each exchange
poloniex = None
for _, exchange in self.exchange_manager.connected_exchanges.items():
if exchange.name == 'poloniex':
poloniex = exchange
if not exchange.check_trades_cache(start_ts, end_at_least_ts):
raise HistoryCacheInvalid(f'{exchange.name} cache is invalid')
# Poloniex specific
loan_data = []
if poloniex:
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(
loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts,
)
if not loan_history_is_okay:
raise HistoryCacheInvalid('Poloniex loan cache is invalid')
loan_data = loan_file_contents['data']
asset_movements = self._get_cached_asset_movements(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
eth_tx_log_path)
eth_tx_log_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts,
)
if not eth_tx_log_history_is_okay:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
try:
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts,
)
except KeyError:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
# make sure that this is the same as what is returned
# from create_history, except for the first argument
return (
history_trades,
loan_data,
asset_movements,
eth_transactions,
)
def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[MarginPosition] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[AssetMovement] = None,
eth_transaction_list: List[Dict] = None,
defi_events_list: List[DefiEvent] = None,
ledger_actions_list: List[LedgerAction] = None,
) -> Tuple[Dict[str, Any], List[Dict[str, Any]]]:
trade_history: Sequence[Union[Trade, MarginPosition]]
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=Timestamp(0),
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
# if present, append margin positions to trade history
if margin_list:
trade_history.extend(margin_list) # type: ignore
asset_movements = []
if asset_movements_list:
asset_movements = asset_movements_list
loan_history = []
if loans_list:
loan_history = process_polo_loans(
msg_aggregator=accountant.msg_aggregator,
data=loans_list,
start_ts=Timestamp(0),
end_ts=end_ts,
)
eth_transactions = []
if eth_transaction_list:
eth_transactions = eth_transaction_list
defi_events = []
if defi_events_list:
defi_events = defi_events_list
ledger_actions = []
if ledger_actions_list:
ledger_actions = ledger_actions_list
report_id = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
defi_events=defi_events,
ledger_actions=ledger_actions,
)
dbpnl = DBAccountingReports(accountant.csvexporter.database)
report = dbpnl.get_reports(report_id=report_id, with_limit=False)[0][0]
events = dbpnl.get_report_data(
filter_=ReportDataFilterQuery.make(report_id=1),
with_limit=False,
)[0]
return report, events