def populate_history_cb(
result_history: Union[List[Trade], List[MarginPosition]],
result_asset_movements: List[AssetMovement],
exchange_specific_data: Any,
) -> None:
"""This callback will run for succesfull exchange history query"""
history.extend(result_history)
asset_movements.extend(result_asset_movements)
if exchange_specific_data:
# This can only be poloniex at the moment
polo_loans_data = exchange_specific_data
polo_loans.extend(
process_polo_loans(
msg_aggregator=self.msg_aggregator,
data=polo_loans_data,
start_ts=start_ts,
end_ts=end_ts,
))
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
write_history_data_in_file(polo_loans, loansfile_path,
start_ts, end_ts)
def get_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> List[PriceHistoryEntry]:
"""
Get historical price data from cryptocompare
Returns a sorted list of price entries.
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
cache_key = PairCacheKey(from_asset.identifier + '_' +
to_asset.identifier)
got_cached_value = self._got_cached_price(cache_key, timestamp)
if got_cached_value:
return self.price_history[cache_key].data
now_ts = ts_now()
cryptocompare_hourquerylimit = 2000
calculated_history: List[Dict[str, Any]] = []
if historical_data_start <= timestamp:
end_date = historical_data_start
else:
end_date = timestamp
while True:
pr_end_date = end_date
end_date = Timestamp(end_date +
(cryptocompare_hourquerylimit) * 3600)
log.debug(
'Querying cryptocompare for hourly historical price',
from_asset=from_asset,
to_asset=to_asset,
cryptocompare_hourquerylimit=cryptocompare_hourquerylimit,
end_date=end_date,
)
resp = self.query_endpoint_histohour(
from_asset=from_asset,
to_asset=to_asset,
limit=2000,
to_timestamp=end_date,
)
if pr_end_date != resp['TimeFrom']:
# If we get more than we needed, since we are close to the now_ts
# then skip all the already included entries
diff = pr_end_date - resp['TimeFrom']
# If the start date has less than 3600 secs difference from previous
# end date then do nothing. If it has more skip all already included entries
if diff >= 3600:
if resp['Data'][diff // 3600]['time'] != pr_end_date:
raise RemoteError(
'Unexpected fata format in cryptocompare query_endpoint_histohour. '
'Expected to find the previous date timestamp during '
'cryptocompare historical data fetching', )
# just add only the part from the previous timestamp and on
resp['Data'] = resp['Data'][diff // 3600:]
# The end dates of a cryptocompare query do not match. The end date
# can have up to 3600 secs different to the requested one since this is
# hourly historical data but no more.
end_dates_dont_match = (end_date < now_ts
and resp['TimeTo'] != end_date)
if end_dates_dont_match:
if resp['TimeTo'] - end_date >= 3600:
raise RemoteError(
'Unexpected fata format in cryptocompare query_endpoint_histohour. '
'End dates do not match.', )
else:
# but if it's just a drift within an hour just update the end_date so that
# it can be picked up by the next iterations in the loop
end_date = resp['TimeTo']
# If last time slot and first new are the same, skip the first new slot
last_entry_equal_to_first = (len(calculated_history) != 0
and calculated_history[-1]['time']
== resp['Data'][0]['time'])
if last_entry_equal_to_first:
resp['Data'] = resp['Data'][1:]
calculated_history += resp['Data']
if end_date >= now_ts:
break
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# and now since we actually queried the data let's also cache them
filename = self.data_directory / ('price_history_' + cache_key +
'.json')
log.info(
'Updating price history cache',
filename=filename,
from_asset=from_asset,
to_asset=to_asset,
)
write_history_data_in_file(
data=calculated_history,
filepath=filename,
start_ts=historical_data_start,
end_ts=now_ts,
)
# Finally save the objects in memory and return them
data_including_time = {
'data': calculated_history,
'start_time': historical_data_start,
'end_time': end_date,
}
self.price_history_file[cache_key] = filename
self.price_history[cache_key] = _dict_history_to_data(
data_including_time)
return self.price_history[cache_key].data
def create_history(self, start_ts, end_ts, end_at_least_ts):
"""Creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history for that particular source
which satisfies it we return the cache
"""
log.info(
'Starting trade history creation',
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
# start creating the all trades history list
history = list()
asset_movements = list()
empty_or_error = ''
if self.kraken is not None:
try:
kraken_history = self.kraken.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(kraken_history)
kraken_asset_movements = self.kraken.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(kraken_asset_movements)
except RemoteError as e:
empty_or_error += '\n' + str(e)
poloniex_query_error = False
poloniex_margin_trades = []
polo_loans = []
try:
(
history,
asset_movements,
poloniex_margin_trades,
polo_loans,
) = self.query_poloniex_history(
history,
asset_movements,
start_ts,
end_ts,
end_at_least_ts,
)
except RemoteError as e:
empty_or_error += '\n' + str(e)
poloniex_query_error = True
if self.bittrex is not None:
try:
bittrex_history = self.bittrex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(bittrex_history)
except RemoteError as e:
empty_or_error += '\n' + str(e)
if self.bitmex is not None:
try:
bitmex_history = self.bitmex.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
for trade in bitmex_history:
history.append(trade_from_bitmex(trade))
bitmex_asset_movements = self.bitmex.query_deposits_withdrawals(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
asset_movements.extend(bitmex_asset_movements)
except RemoteError as e:
empty_or_error += '\n' + str(e)
if self.binance is not None:
try:
binance_history = self.binance.query_trade_history(
start_ts=start_ts,
end_ts=end_ts,
end_at_least_ts=end_at_least_ts,
)
history.extend(binance_history)
except RemoteError as e:
empty_or_error += '\n' + str(e)
try:
eth_transactions = query_etherscan_for_transactions(
self.eth_accounts)
except RemoteError as e:
empty_or_error += '\n' + str(e)
# We sort it here ... but when accounting runs through the entire actions list,
# it resorts, so unless the fact that we sort is used somewhere else too, perhaps
# we can skip it?
history.sort(key=lambda trade: trade.timestamp)
history = limit_trade_list_to_period(history, start_ts, end_ts)
# Write to files
historyfile_path = os.path.join(self.user_directory,
TRADES_HISTORYFILE)
write_tupledata_history_in_file(history, historyfile_path, start_ts,
end_ts)
if self.poloniex is not None:
if not self.read_manual_margin_positions and not poloniex_query_error:
marginfile_path = os.path.join(self.user_directory,
MARGIN_HISTORYFILE)
write_tupledata_history_in_file(
poloniex_margin_trades,
marginfile_path,
start_ts,
end_ts,
)
if not poloniex_query_error:
loansfile_path = os.path.join(self.user_directory,
LOANS_HISTORYFILE)
write_history_data_in_file(polo_loans, loansfile_path, start_ts,
end_ts)
assetmovementsfile_path = os.path.join(self.user_directory,
ASSETMOVEMENTS_HISTORYFILE)
write_tupledata_history_in_file(asset_movements,
assetmovementsfile_path, start_ts,
end_ts)
eth_tx_log_path = os.path.join(self.user_directory,
ETHEREUM_TX_LOGFILE)
write_tupledata_history_in_file(eth_transactions, eth_tx_log_path,
start_ts, end_ts)
# After writting everything to files include the external trades in the history
history = maybe_add_external_trades_to_history(
db=self.db,
start_ts=start_ts,
end_ts=end_ts,
history=history,
msg_aggregator=self.msg_aggregator,
)
return (
empty_or_error,
history,
poloniex_margin_trades,
polo_loans,
asset_movements,
eth_transactions,
)
def write_tupledata_history_in_file(history, filepath, start_ts, end_ts):
out_history = [tr._asdict() for tr in history]
write_history_data_in_file(out_history, filepath, start_ts, end_ts)
def get_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
only_check_cache: bool,
) -> Optional[List[PriceHistoryEntry]]:
"""
Get historical hour price data from cryptocompare
Returns a sorted list of price entries.
If only_check_cache is True then if the data is not cached locally this will return None
- May raise RemoteError if there is a problem reaching the cryptocompare server
or with reading the response returned by the server
- May raise UnsupportedAsset if from/to asset is not supported by cryptocompare
"""
log.debug(
'Retrieving historical price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
cache_key = PairCacheKey(from_asset.identifier + '_' +
to_asset.identifier)
cached_data = self._got_cached_data_at_timestamp(
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if cached_data is not None:
return cached_data.data
if only_check_cache:
return None
now_ts = ts_now()
# save time at start of the query, in case the query does not complete due to rate ilmit
self.last_histohour_query_ts = now_ts
if cache_key in self.price_history:
old_data = self.price_history[cache_key].data
if timestamp > self.price_history[cache_key].end_time:
# We have a cache but the requested timestamp does not hit it
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=self.price_history[cache_key].end_time,
)
if old_data[-1].time == new_data[0]['time']:
old_data = old_data[:-1]
new_history = deque([x._asdict() for x in old_data]) + new_data
else:
# only other possibility, timestamp < cached start_time
# Get all available data, even before to_timestamp
new_data = self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=self.price_history[cache_key].start_time,
to_timestamp=Timestamp(0),
)
if new_data[-1]['time'] == old_data[0].time:
new_data.pop()
new_history = new_data + deque([x._asdict() for x in old_data])
calculated_history = list(new_history)
else:
calculated_history = list(
self._get_histohour_data_for_range(
from_asset=from_asset,
to_asset=to_asset,
from_timestamp=now_ts,
to_timestamp=Timestamp(0),
))
if len(calculated_history) == 0:
return [] # empty list means we found nothing
# Let's always check for data sanity for the hourly prices.
_check_hourly_data_sanity(calculated_history, from_asset, to_asset)
# and now since we actually queried the data let's also cache them
filename = (get_or_make_price_history_dir(self.data_directory) /
(PRICE_HISTORY_FILE_PREFIX + cache_key + '.json'))
log.info(
'Updating price history cache',
filename=filename,
from_asset=from_asset,
to_asset=to_asset,
)
write_history_data_in_file(
data=calculated_history,
filepath=filename,
start_ts=calculated_history[0]['time'],
end_ts=now_ts,
)
# Finally save the objects in memory and return them
data_including_time = {
'data': calculated_history,
'start_time': calculated_history[0]['time'],
'end_time': now_ts,
}
self.price_history_file[cache_key] = filename
self.price_history[cache_key] = _dict_history_to_data(
data_including_time)
self.last_histohour_query_ts = ts_now(
) # also save when last query finished
return self.price_history[cache_key].data
def get_historical_data(
self,
from_asset: Asset,
to_asset: Asset,
timestamp: Timestamp,
historical_data_start: Timestamp,
) -> List[PriceHistoryEntry]:
"""
Get historical price data from cryptocompare
Returns a sorted list of price entries.
"""
log.debug(
'Retrieving historical price data from cryptocompare',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
cache_key = PairCacheKey(from_asset.identifier + '_' +
to_asset.identifier)
got_cached_value = self.got_cached_price(cache_key, timestamp)
if got_cached_value:
return self.price_history[cache_key].data
now_ts = int(time.time())
cryptocompare_hourquerylimit = 2000
calculated_history: List = list()
if historical_data_start <= timestamp:
end_date = historical_data_start
else:
end_date = timestamp
while True:
pr_end_date = end_date
end_date = Timestamp(end_date +
(cryptocompare_hourquerylimit) * 3600)
log.debug(
'Querying cryptocompare for hourly historical price',
from_asset=from_asset,
to_asset=to_asset,
cryptocompare_hourquerylimit=cryptocompare_hourquerylimit,
end_date=end_date,
)
resp = self.query_endpoint_histohour(
from_asset=from_asset,
to_asset=to_asset,
limit=2000,
to_timestamp=end_date,
)
if pr_end_date != resp['TimeFrom']:
# If we get more than we needed, since we are close to the now_ts
# then skip all the already included entries
diff = pr_end_date - resp['TimeFrom']
if resp['Data'][diff // 3600]['time'] != pr_end_date:
raise ValueError(
'Expected to find the previous date timestamp during '
'historical data fetching', )
# just add only the part from the previous timestamp and on
resp['Data'] = resp['Data'][diff // 3600:]
if end_date < now_ts and resp['TimeTo'] != end_date:
raise ValueError('End dates no match')
# If last time slot and first new are the same, skip the first new slot
last_entry_equal_to_first = (len(calculated_history) != 0
and calculated_history[-1]['time']
== resp['Data'][0]['time'])
if last_entry_equal_to_first:
resp['Data'] = resp['Data'][1:]
calculated_history += resp['Data']
if end_date >= now_ts:
break
# Let's always check for data sanity for the hourly prices.
assert _check_hourly_data_sanity(calculated_history, from_asset,
to_asset)
# and now since we actually queried the data let's also cache them
filename = FilePath(
os.path.join(self.data_directory,
'price_history_' + cache_key + '.json'), )
log.info(
'Updating price history cache',
filename=filename,
from_asset=from_asset,
to_asset=to_asset,
)
write_history_data_in_file(
data=calculated_history,
filepath=filename,
start_ts=historical_data_start,
end_ts=now_ts,
)
# Finally save the objects in memory and return them
data_including_time = {
'data': calculated_history,
'start_time': historical_data_start,
'end_time': end_date,
}
self.price_history_file[cache_key] = filename
self.price_history[cache_key] = _dict_history_to_data(
data_including_time)
return self.price_history[cache_key].data
