def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[MarginPosition] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[Dict] = None,
eth_transaction_list: List[Dict] = None,
) -> Dict[str, Any]:
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=0,
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
# if present, append margin positions to trade history
if margin_list:
trade_history.extend(margin_list)
asset_movements = list()
if asset_movements_list:
asset_movements = asset_movements_from_dictlist(
given_data=asset_movements_list,
start_ts=0,
end_ts=end_ts,
)
loan_history = list()
if loans_list:
loan_history = process_polo_loans(
msg_aggregator=accountant.msg_aggregator,
data=loans_list,
start_ts=0,
end_ts=end_ts,
)
eth_transactions = list()
if eth_transaction_list:
eth_transactions = transactions_from_dictlist(
given_transactions=eth_transaction_list,
start_ts=0,
end_ts=end_ts,
)
result = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result
def accounting_history_process(
accountant,
start_ts: Timestamp,
end_ts: Timestamp,
history_list: List[Dict],
margin_list: List[Dict] = None,
loans_list: List[Dict] = None,
asset_movements_list: List[Dict] = None,
eth_transaction_list: List[Dict] = None,
) -> Dict[str, Any]:
# For filtering the taxable actions list we start with 0 ts so that we have the
# full history available
trade_history = trades_from_dictlist(
given_trades=history_list,
start_ts=0,
end_ts=end_ts,
location='accounting_history_process for tests',
msg_aggregator=accountant.msg_aggregator,
)
margin_history = [] if not margin_list else margin_list
loan_history = [] if not loans_list else loans_list
asset_movements = list()
if asset_movements_list:
asset_movements = asset_movements_from_dictlist(
given_data=asset_movements_list,
start_ts=0,
end_ts=end_ts,
)
eth_transactions = list()
if eth_transaction_list:
eth_transactions = transactions_from_dictlist(
given_transactions=eth_transaction_list,
start_ts=0,
end_ts=end_ts,
)
result = accountant.process_history(
start_ts=start_ts,
end_ts=end_ts,
trade_history=trade_history,
margin_history=margin_history,
loan_history=loan_history,
asset_movements=asset_movements,
eth_transactions=eth_transactions,
)
return result
def get_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets or creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history which satisfies it we
return the cache
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.data_directory, TRADES_HISTORYFILE)
if os.path.isfile(historyfile_path):
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except:
pass
all_history_okay = data_up_todate(history_json_data, start_ts, end_at_least_ts)
poloniex_history_okay = True
if self.poloniex is not None:
poloniex_history_okay = self.poloniex.check_trades_cache(
start_ts, end_at_least_ts
) is not None
kraken_history_okay = True
if self.kraken is not None:
kraken_history_okay = self.kraken.check_trades_cache(
start_ts, end_at_least_ts
) is not None
bittrex_history_okay = True
if self.bittrex is not None:
bittrex_history_okay = self.bittrex.check_trades_cache(
start_ts, end_at_least_ts
) is not None
binance_history_okay = True
if self.binance is not None:
binance_history_okay = self.binance.check_trades_cache(
start_ts, end_at_least_ts
) is not None
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.data_directory, MARGIN_HISTORYFILE)
margin_file_contents = get_jsonfile_contents_or_empty_dict(marginfile_path)
margin_history_is_okay = data_up_todate(
margin_file_contents,
start_ts,
end_at_least_ts
)
else:
margin_history_is_okay = True
margin_file_contents = do_read_manual_margin_positions(
self.data_directory
)
loansfile_path = os.path.join(self.data_directory, LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts
)
assetmovementsfile_path = os.path.join(
self.data_directory,
ASSETMOVEMENTS_HISTORYFILE
)
asset_movements_contents = get_jsonfile_contents_or_empty_dict(
assetmovementsfile_path
)
asset_movements_history_is_okay = data_up_todate(
asset_movements_contents,
start_ts,
end_at_least_ts
)
eth_tx_log_path = os.path.join(self.data_directory, ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(eth_tx_log_path)
eth_tx_log_history_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts
)
if (
all_history_okay and
poloniex_history_okay and
kraken_history_okay and
bittrex_history_okay and
binance_history_okay and
margin_history_is_okay and
loan_history_is_okay and
asset_movements_history_is_okay and
eth_tx_log_history_history_is_okay):
history_trades = trades_from_dictlist(
history_json_data['data'],
start_ts,
end_ts
)
if not self.read_manual_margin_positions:
margin_trades = trades_from_dictlist(
margin_file_contents['data'],
start_ts,
end_ts
)
else:
margin_trades = margin_file_contents
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts
)
asset_movements = asset_movements_from_dictlist(
asset_movements_contents['data'],
start_ts,
end_ts
)
history_trades = include_external_trades(
self.db,
start_ts,
end_ts,
history_trades
)
# make sure that this is the same as what is returned
# from create_history
return (
history_trades,
margin_trades,
loan_file_contents['data'],
asset_movements,
eth_transactions
)
return self.create_history(start_ts, end_ts, end_at_least_ts)
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets all the cached history data instead of querying all external sources
to create the history through create_history()
Can raise:
- HistoryCacheInvalid:
If any of the cache files are corrupt in any way, missing or
do not cover the given time range
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
if not os.path.isfile(historyfile_path):
raise HistoryCacheInvalid()
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except JSONDecodeError:
pass
if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Historical trades cache invalid')
try:
history_trades = trades_from_dictlist(
given_trades=history_json_data['data'],
start_ts=start_ts,
end_ts=end_ts,
location='historical trades',
msg_aggregator=self.msg_aggregator,
)
except KeyError:
raise HistoryCacheInvalid('Historical trades cache invalid')
history_trades = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history_trades,
msg_aggregator=self.msg_aggregator,
)
kraken_okay = (self.kraken is None or self.kraken.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not kraken_okay:
raise HistoryCacheInvalid('Kraken cache is invalid')
bittrex_okay = (self.bittrex is None
or self.bittrex.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not bittrex_okay:
raise HistoryCacheInvalid('Bittrex cache is invalid')
binance_okay = (self.binance is None
or self.binance.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not binance_okay:
raise HistoryCacheInvalid('Binance cache is invalid')
bitmex_okay = (self.bitmex is None or self.bitmex.check_trades_cache(
start_ts,
end_at_least_ts,
) is not None)
if not bitmex_okay:
raise HistoryCacheInvalid('Bitmex cache is invalid')
# Poloniex specific
loan_data = []
if self.poloniex:
if not self.poloniex.check_trades_cache(start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Poloniex cache is invalid')
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(
loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts,
)
if not loan_history_is_okay:
raise HistoryCacheInvalid('Poloniex loan cache is invalid')
loan_data = loan_file_contents['data']
# margin positions that have been manually input
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.user_directory,
MARGIN_HISTORYFILE)
margin_file_contents = get_jsonfile_contents_or_empty_dict(
marginfile_path)
margin_history_is_okay = data_up_todate(
margin_file_contents,
start_ts,
end_at_least_ts,
)
if not margin_history_is_okay:
raise HistoryCacheInvalid('Margin Positions cache is invalid')
try:
margin_trades = trades_from_dictlist(
given_trades=margin_file_contents['data'],
start_ts=start_ts,
end_ts=end_ts,
location='Margin position trades',
msg_aggregator=self.msg_aggregator,
)
except KeyError:
raise HistoryCacheInvalid('Margin Positions cache is invalid')
else:
margin_trades = do_read_manual_margin_positions(
self.user_directory, )
asset_movements = self._get_cached_asset_movements(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
eth_tx_log_path)
eth_tx_log_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts,
)
if not eth_tx_log_history_is_okay:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
try:
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts,
)
except KeyError:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
# make sure that this is the same as what is returned
# from create_history, except for the first argument
return (
history_trades,
margin_trades,
loan_data,
asset_movements,
eth_transactions,
)
def get_cached_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets all the cached history data instead of querying all external sources
to create the history through create_history()
Can raise:
- HistoryCacheInvalid:
If any of the cache files are corrupt in any way, missing or
do not cover the given time range
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
if not os.path.isfile(historyfile_path):
raise HistoryCacheInvalid()
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except JSONDecodeError:
pass
if not data_up_todate(history_json_data, start_ts, end_at_least_ts):
raise HistoryCacheInvalid('Historical trades cache invalid')
try:
history_trades = trades_from_dictlist(
given_trades=history_json_data['data'],
start_ts=start_ts,
end_ts=end_ts,
location='historical trades',
msg_aggregator=self.msg_aggregator,
)
except (KeyError, DeserializationError):
raise HistoryCacheInvalid('Historical trades cache invalid')
history_trades = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history_trades,
msg_aggregator=self.msg_aggregator,
)
# Check the cache of each exchange
poloniex = None
for _, exchange in self.exchange_manager.connected_exchanges.items():
if exchange.name == 'poloniex':
poloniex = exchange
if not exchange.check_trades_cache(start_ts, end_at_least_ts):
raise HistoryCacheInvalid(f'{exchange.name} cache is invalid')
# Poloniex specific
loan_data = []
if poloniex:
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(
loansfile_path)
loan_history_is_okay = data_up_todate(
loan_file_contents,
start_ts,
end_at_least_ts,
)
if not loan_history_is_okay:
raise HistoryCacheInvalid('Poloniex loan cache is invalid')
loan_data = loan_file_contents['data']
asset_movements = self._get_cached_asset_movements(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
eth_tx_log_path)
eth_tx_log_history_is_okay = data_up_todate(
eth_tx_log_contents,
start_ts,
end_at_least_ts,
)
if not eth_tx_log_history_is_okay:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
try:
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'],
start_ts,
end_ts,
)
except KeyError:
raise HistoryCacheInvalid('Ethereum transactions cache is invalid')
# make sure that this is the same as what is returned
# from create_history, except for the first argument
return (
history_trades,
loan_data,
asset_movements,
eth_transactions,
)