def include_external_trades(db, start_ts, end_ts, history):
external_trades = db.get_external_trades()
external_trades = trades_from_dictlist(external_trades, start_ts, end_ts)
history.extend(external_trades)
history.sort(key=lambda trade: trade.timestamp)
return history
def maybe_add_external_trades_to_history(
db: DBHandler,
start_ts: Timestamp,
end_ts: Timestamp,
history: List[Trade],
msg_aggregator: MessagesAggregator,
) -> List[Trade]:
"""
Queries the DB to get any external trades, adds them to the provided history and returns it.
If there is an unexpected error at the external trade deserialization an error is logged.
"""
serialized_external_trades = db.get_trades()
try:
external_trades = trades_from_dictlist(
given_trades=serialized_external_trades,
start_ts=start_ts,
end_ts=end_ts,
location='external trades',
msg_aggregator=msg_aggregator,
)
except KeyError:
msg_aggregator.add_error(
'External trades in the DB are in an unrecognized format')
return history
history.extend(external_trades)
history.sort(key=lambda trade: trade.timestamp)
return history
def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[Dict] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[Dict] = None,
eth_transaction_list: List[Dict] = None,
) -> Dict[str, Any]:
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=0,
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
margin_history = [] if not margin_list else margin_list
asset_movements = list()
if asset_movements_list:
asset_movements = asset_movements_from_dictlist(
given_data=asset_movements_list,
start_ts=0,
end_ts=end_ts,
)
loan_history = list()
if loans_list:
loan_history = process_polo_loans(
msg_aggregator=accountant.msg_aggregator,
data=loans_list,
start_ts=0,
end_ts=end_ts,
)
eth_transactions = list()
if eth_transaction_list:
eth_transactions = transactions_from_dictlist(
given_transactions=eth_transaction_list,
start_ts=0,
end_ts=end_ts,
)
result = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
margin_history=margin_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result
def test_trades_from_dictlist(function_scope_messages_aggregator):
raw_trades = [raw_trade1, raw_trade2, raw_trade3]
trades = trades_from_dictlist(
given_trades=raw_trades,
start_ts=1516985747,
end_ts=1557985736,
location='test',
msg_aggregator=function_scope_messages_aggregator,
)
assert len(trades) == 1
assert isinstance(trades[0], Trade)
def get_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets or creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history which satisfies it we
return the cache
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.data_directory, TRADES_HISTORYFILE)
if os.path.isfile(historyfile_path):
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except:
pass
all_history_okay = data_up_todate(history_json_data, start_ts, end_at_least_ts)
poloniex_history_okay = True
if self.poloniex is not None:
poloniex_history_okay = self.poloniex.check_trades_cache(
start_ts, end_at_least_ts
) is not None
kraken_history_okay = True
if self.kraken is not None:
kraken_history_okay = self.kraken.check_trades_cache(
start_ts, end_at_least_ts
) is not None
bittrex_history_okay = True
if self.bittrex is not None:
bittrex_history_okay = self.bittrex.check_trades_cache(
start_ts, end_at_least_ts
) is not None
binance_history_okay = True
if self.binance is not None:
binance_history_okay = self.binance.check_trades_cache(
start_ts, end_at_least_ts
) is not None
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.data_directory, MARGIN_HISTORYFILE)
margin_file_contents = get_jsonfile_contents_or_empty_dict(marginfile_path)
margin_history_is_okay = data_up_todate(
margin_file_contents,
start_ts,
end_at_least_ts
)
else:
margin_history_is_okay = True
margin_file_contents = do_read_manual_margin_positions(
self.data_directory
)
loansfile_path = os.path.join(self.data_directory, LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts
)
assetmovementsfile_path = os.path.join(
self.data_directory,
ASSETMOVEMENTS_HISTORYFILE
)
asset_movements_contents = get_jsonfile_contents_or_empty_dict(
assetmovementsfile_path
)
asset_movements_history_is_okay = data_up_todate(
asset_movements_contents,
start_ts,
end_at_least_ts
)
eth_tx_log_path = os.path.join(self.data_directory, ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(eth_tx_log_path)
eth_tx_log_history_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts
)
if (
all_history_okay and
poloniex_history_okay and
kraken_history_okay and
bittrex_history_okay and
binance_history_okay and
margin_history_is_okay and
loan_history_is_okay and
asset_movements_history_is_okay and
eth_tx_log_history_history_is_okay):
history_trades = trades_from_dictlist(
history_json_data['data'],
start_ts,
end_ts
)
if not self.read_manual_margin_positions:
margin_trades = trades_from_dictlist(
margin_file_contents['data'],
start_ts,
end_ts
)
else:
margin_trades = margin_file_contents
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts
)
asset_movements = asset_movements_from_dictlist(
asset_movements_contents['data'],
start_ts,
end_ts
)
history_trades = include_external_trades(
self.db,
start_ts,
end_ts,
history_trades
)
# make sure that this is the same as what is returned
# from create_history
return (
history_trades,
margin_trades,
loan_file_contents['data'],
asset_movements,
eth_transactions
)
return self.create_history(start_ts, end_ts, end_at_least_ts)
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets all the cached history data instead of querying all external sources
to create the history through create_history()
Can raise:
- HistoryCacheInvalid:
If any of the cache files are corrupt in any way, missing or
do not cover the given time range
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
if not os.path.isfile(historyfile_path):
raise HistoryCacheInvalid()
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except JSONDecodeError:
pass
if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Historical trades cache invalid')
try:
history_trades = trades_from_dictlist(
given_trades=history_json_data['data'],
start_ts=start_ts,
end_ts=end_ts,
location='historical trades',
msg_aggregator=self.msg_aggregator,
)
except KeyError:
raise HistoryCacheInvalid('Historical trades cache invalid')
history_trades = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history_trades,
msg_aggregator=self.msg_aggregator,
)
kraken_okay = (self.kraken is None or self.kraken.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not kraken_okay:
raise HistoryCacheInvalid('Kraken cache is invalid')
bittrex_okay = (self.bittrex is None
or self.bittrex.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not bittrex_okay:
raise HistoryCacheInvalid('Bittrex cache is invalid')
binance_okay = (self.binance is None
or self.binance.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not binance_okay:
raise HistoryCacheInvalid('Binance cache is invalid')
bitmex_okay = (self.bitmex is None or self.bitmex.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not bitmex_okay:
raise HistoryCacheInvalid('Bitmex cache is invalid')
# Poloniex specific
loan_data = []
if self.poloniex:
if not self.poloniex.check_trades_cache(start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Poloniex cache is invalid')
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(
loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts,
)
if not loan_history_is_okay:
raise HistoryCacheInvalid('Poloniex loan cache is invalid')
loan_data = loan_file_contents['data']
# margin positions that have been manually input
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.user_directory,
MARGIN_HISTORYFILE)
margin_file_contents = get_jsonfile_contents_or_empty_dict(
marginfile_path)
margin_history_is_okay = data_up_todate(
margin_file_contents,
start_ts,
end_at_least_ts,
)
if not margin_history_is_okay:
raise HistoryCacheInvalid('Margin Positions cache is invalid')
try:
margin_trades = trades_from_dictlist(
given_trades=margin_file_contents['data'],
start_ts=start_ts,
end_ts=end_ts,
location='Margin position trades',
msg_aggregator=self.msg_aggregator,
)
except KeyError:
raise HistoryCacheInvalid('Margin Positions cache is invalid')
else:
margin_trades = do_read_manual_margin_positions(
self.user_directory, )
asset_movements = self._get_cached_asset_movements(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
eth_tx_log_path)
eth_tx_log_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts,
)
if not eth_tx_log_history_is_okay:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
try:
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts,
)
except KeyError:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
# make sure that this is the same as what is returned
# from create_history, except for the first argument
return (
history_trades,
margin_trades,
loan_data,
asset_movements,
eth_transactions,
)
def test_trades_from_dictlist():
raw_trades = [raw_trade1, raw_trade2, raw_trade3]
trades = trades_from_dictlist(raw_trades, 1516985747, 1557985736)
assert len(trades) == 1
assert isinstance(trades[0], Trade)
