def create_history(self, start_ts, end_ts, end_at_least_ts):
"""Creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history for that particular source
which satisfies it we return the cache
"""
# start creating the all trades history list
history = list()
asset_movements = list()
if self.kraken is not None:
kraken_history = self.kraken.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts
)
for trade in kraken_history:
history.append(trade_from_kraken(trade))
kraken_asset_movements = self.kraken.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(kraken_asset_movements)
if self.poloniex is not None:
polo_history = self.poloniex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts
)
poloniex_margin_trades = list()
for pair, trades in polo_history.items():
for trade in trades:
category = trade['category']
if category == 'exchange' or category == 'settlement':
history.append(trade_from_poloniex(trade, pair))
elif category == 'marginTrade':
if not self.read_manual_margin_positions:
poloniex_margin_trades.append(trade_from_poloniex(trade, pair))
else:
raise ValueError("Unexpected poloniex trade category: {}".format(category))
if self.read_manual_margin_positions:
# Just read the manual positions log and make virtual trades that
# correspond to the profits
assert poloniex_margin_trades == list(), (
"poloniex margin trades list should be empty here"
)
poloniex_margin_trades = do_read_manual_margin_positions(
self.data_directory
)
else:
poloniex_margin_trades.sort(key=lambda trade: trade.timestamp)
poloniex_margin_trades = limit_trade_list_to_period(
poloniex_margin_trades,
start_ts,
end_ts
)
polo_loans = self.poloniex.query_loan_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
from_csv=True
)
polo_loans = process_polo_loans(polo_loans, start_ts, end_ts)
polo_asset_movements = self.poloniex.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(polo_asset_movements)
if self.bittrex is not None:
bittrex_history = self.bittrex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts
)
for trade in bittrex_history:
history.append(trade_from_bittrex(trade))
if self.binance is not None:
binance_history = self.binance.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts
)
for trade in binance_history:
history.append(trade_from_binance(trade))
eth_transactions = query_etherscan_for_transactions(self.eth_accounts)
# We sort it here ... but when accounting runs through the entire actions list,
# it resorts, so unless the fact that we sort is used somewhere else too, perhaps
# we can skip it?
history.sort(key=lambda trade: trade.timestamp)
history = limit_trade_list_to_period(history, start_ts, end_ts)
# Write to files
historyfile_path = os.path.join(self.data_directory, TRADES_HISTORYFILE)
write_tupledata_history_in_file(history, historyfile_path, start_ts, end_ts)
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.data_directory, MARGIN_HISTORYFILE)
write_tupledata_history_in_file(
poloniex_margin_trades,
marginfile_path,
start_ts,
end_ts
)
loansfile_path = os.path.join(self.data_directory, LOANS_HISTORYFILE)
write_history_data_in_file(polo_loans, loansfile_path, start_ts, end_ts)
assetmovementsfile_path = os.path.join(self.data_directory, ASSETMOVEMENTS_HISTORYFILE)
write_tupledata_history_in_file(asset_movements, assetmovementsfile_path, start_ts, end_ts)
eth_tx_log_path = os.path.join(self.data_directory, ETHEREUM_TX_LOGFILE)
write_tupledata_history_in_file(eth_transactions, eth_tx_log_path, start_ts, end_ts)
# After writting everything to files include the external trades in the history
history = include_external_trades(self.db, start_ts, end_ts, history)
return history, poloniex_margin_trades, polo_loans, asset_movements, eth_transactions
def create_history(self, start_ts: Timestamp, end_ts: Timestamp,
end_at_least_ts: Timestamp):
"""Creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history for that particular source
which satisfies it we return the cache
"""
log.info(
'Starting trade history creation',
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
# start creating the all trades history list
history: List[Union[Trade, MarginPosition]] = list()
asset_movements = list()
polo_loans = list()
empty_or_error = ''
def populate_history_cb(
result_history: Union[List[Trade], List[MarginPosition]],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(result_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
polo_loans.extend(
process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
start_ts=start_ts,
end_ts=end_ts,
))
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
write_history_data_in_file(polo_loans, loansfile_path,
start_ts, end_ts)
def fail_history_cb(error_msg: str) -> None:
"""This callback will run for failure in exchange history query"""
nonlocal empty_or_error
empty_or_error += '\n' + error_msg
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange.query_history_with_callbacks(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
success_callback=populate_history_cb,
fail_callback=fail_history_cb,
)
try:
eth_transactions = query_etherscan_for_transactions(
self.eth_accounts)
except RemoteError as e:
empty_or_error += '\n' + str(e)
# We sort it here ... but when accounting runs through the entire actions list,
# it resorts, so unless the fact that we sort is used somewhere else too, perhaps
# we can skip it?
history.sort(key=lambda trade: action_get_timestamp(trade))
history = limit_trade_list_to_period(history, start_ts, end_ts)
# Write to files
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
write_tupledata_history_in_file(history, historyfile_path, start_ts,
end_ts)
assetmovementsfile_path = os.path.join(self.user_directory,
ASSETMOVEMENTS_HISTORYFILE)
write_tupledata_history_in_file(asset_movements,
assetmovementsfile_path, start_ts,
end_ts)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
write_tupledata_history_in_file(eth_transactions, eth_tx_log_path,
start_ts, end_ts)
# After writting everything to files include the external trades in the history
history = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history,
msg_aggregator=self.msg_aggregator,
)
return (
empty_or_error,
history,
polo_loans,
asset_movements,
eth_transactions,
)
def create_history(self, start_ts, end_ts, end_at_least_ts):
"""Creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history for that particular source
which satisfies it we return the cache
"""
log.info(
'Starting trade history creation',
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
# start creating the all trades history list
history = list()
asset_movements = list()
empty_or_error = ''
if self.kraken is not None:
try:
kraken_history = self.kraken.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(kraken_history)
kraken_asset_movements = self.kraken.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(kraken_asset_movements)
except RemoteError as e:
empty_or_error += '\n' + str(e)
poloniex_query_error = False
poloniex_margin_trades = []
polo_loans = []
try:
(
history,
asset_movements,
poloniex_margin_trades,
polo_loans,
) = self.query_poloniex_history(
history,
asset_movements,
start_ts,
end_ts,
end_at_least_ts,
)
except RemoteError as e:
empty_or_error += '\n' + str(e)
poloniex_query_error = True
if self.bittrex is not None:
try:
bittrex_history = self.bittrex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(bittrex_history)
except RemoteError as e:
empty_or_error += '\n' + str(e)
if self.bitmex is not None:
try:
bitmex_history = self.bitmex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
for trade in bitmex_history:
history.append(trade_from_bitmex(trade))
bitmex_asset_movements = self.bitmex.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(bitmex_asset_movements)
except RemoteError as e:
empty_or_error += '\n' + str(e)
if self.binance is not None:
try:
binance_history = self.binance.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(binance_history)
except RemoteError as e:
empty_or_error += '\n' + str(e)
try:
eth_transactions = query_etherscan_for_transactions(
self.eth_accounts)
except RemoteError as e:
empty_or_error += '\n' + str(e)
# We sort it here ... but when accounting runs through the entire actions list,
# it resorts, so unless the fact that we sort is used somewhere else too, perhaps
# we can skip it?
history.sort(key=lambda trade: trade.timestamp)
history = limit_trade_list_to_period(history, start_ts, end_ts)
# Write to files
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
write_tupledata_history_in_file(history, historyfile_path, start_ts,
end_ts)
if self.poloniex is not None:
if not self.read_manual_margin_positions and not poloniex_query_error:
marginfile_path = os.path.join(self.user_directory,
MARGIN_HISTORYFILE)
write_tupledata_history_in_file(
poloniex_margin_trades,
marginfile_path,
start_ts,
end_ts,
)
if not poloniex_query_error:
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
write_history_data_in_file(polo_loans, loansfile_path, start_ts,
end_ts)
assetmovementsfile_path = os.path.join(self.user_directory,
ASSETMOVEMENTS_HISTORYFILE)
write_tupledata_history_in_file(asset_movements,
assetmovementsfile_path, start_ts,
end_ts)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
write_tupledata_history_in_file(eth_transactions, eth_tx_log_path,
start_ts, end_ts)
# After writting everything to files include the external trades in the history
history = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history,
msg_aggregator=self.msg_aggregator,
)
return (
empty_or_error,
history,
poloniex_margin_trades,
polo_loans,
asset_movements,
eth_transactions,
)
def create_history(self, start_ts: Timestamp, end_ts: Timestamp) -> HistoryResult:
"""Creates trades and loans history from start_ts to end_ts"""
log.info(
'Starting trade history creation',
start_ts=start_ts,
end_ts=end_ts,
)
# start creating the all trades history list
history: List[Union[Trade, MarginPosition]] = list()
asset_movements = list()
polo_loans = list()
empty_or_error = ''
def populate_history_cb(
trades_history: List[Trade],
margin_history: List[MarginPosition],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(trades_history)
history.extend(margin_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
polo_loans.extend(process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
start_ts=start_ts,
end_ts=end_ts,
))
def fail_history_cb(error_msg: str) -> None:
"""This callback will run for failure in exchange history query"""
nonlocal empty_or_error
empty_or_error += '\n' + error_msg
for _, exchange in self.exchange_manager.connected_exchanges.items():
exchange.query_history_with_callbacks(
start_ts=start_ts,
end_ts=end_ts,
success_callback=populate_history_cb,
fail_callback=fail_history_cb,
)
# TODO: Also save those in the DB (?). We used to have a cache but if anything
# makes sense is to save them in the DB. Also realized the old cache broke
# if more accounts were added.
try:
eth_transactions = query_etherscan_for_transactions(
db=self.db,
msg_aggregator=self.msg_aggregator,
from_ts=start_ts,
to_ts=end_ts,
)
except RemoteError as e:
empty_or_error += '\n' + str(e)
# We sort it here ... but when accounting runs through the entire actions list,
# it resorts, so unless the fact that we sort is used somewhere else too, perhaps
# we can skip it?
history.sort(key=lambda trade: action_get_timestamp(trade))
history = limit_trade_list_to_period(history, start_ts, end_ts)
# Include the external trades in the history
external_trades = self.db.get_trades(
from_ts=start_ts,
to_ts=end_ts,
location=Location.EXTERNAL,
)
history.extend(external_trades)
history.sort(key=lambda trade: action_get_timestamp(trade))
return (
empty_or_error,
history,
polo_loans,
asset_movements,
eth_transactions,
)