def transactions_from_dictlist(given_transactions, start_ts, end_ts):
""" Gets a list of transaction, most probably read from the json files and
a time period. Returns it as a list of the transaction tuples that are inside the time period
"""
returned_transactions = list()
for given_tx in given_transactions:
if given_tx['timestamp'] < start_ts:
continue
if given_tx['timestamp'] > end_ts:
break
returned_transactions.append(
EthereumTransaction(
timestamp=convert_to_int(given_tx['timestamp']),
block_number=convert_to_int(given_tx['block_number']),
hash=given_tx['hash'],
from_address=given_tx['from_address'],
to_address=given_tx['to_address'],
value=FVal(given_tx['value']),
gas=FVal(given_tx['gas']),
gas_price=FVal(given_tx['gas_price']),
gas_used=FVal(given_tx['gas_used']),
))
return returned_transactions
def transactions_from_dictlist(
given_transactions: List[Dict],
start_ts: Timestamp,
end_ts: Timestamp,
) -> List[EthereumTransaction]:
""" Gets a list of transaction, most probably read from the json files and
a time period. Returns it as a list of the transaction tuples that are inside the time period
"""
returned_transactions = list()
for given_tx in given_transactions:
if given_tx['timestamp'] < start_ts:
continue
if given_tx['timestamp'] > end_ts:
break
timestamp = Timestamp(convert_to_int(given_tx['timestamp']))
tx_hash = given_tx['hash']
from_address = given_tx['from_address']
to_address = given_tx['to_address']
value = FVal(given_tx['value'])
gas = FVal(given_tx['gas'])
gas_price = FVal(given_tx['gas_price'])
gas_used = FVal(given_tx['gas_used'])
log.debug(
'Processing eth transaction',
sensitive_log=True,
timestamp=timestamp,
eth_tx_hash=tx_hash,
from_eth_address=from_address,
to_eth_address=to_address,
tx_value=value,
gas=gas,
gas_price=gas_price,
gas_used=gas_used,
)
returned_transactions.append(
EthereumTransaction(
timestamp=timestamp,
block_number=convert_to_int(given_tx['block_number']),
hash=tx_hash,
from_address=from_address,
to_address=to_address,
value=value,
gas=gas,
gas_price=gas_price,
gas_used=gas_used,
))
return returned_transactions
def query_ethereum_txlist(
address: EthAddress,
internal: bool,
from_block: int = None,
to_block: int = None,
) -> List[EthereumTransaction]:
result = list()
if internal:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address))
else:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address))
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = request_get(reqstring)
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp))
for v in resp['result']:
# internal tx list contains no gasprice
gas_price = FVal(-1) if internal else FVal(v['gasPrice'])
result.append(
EthereumTransaction(
timestamp=convert_to_int(v['timeStamp']),
block_number=convert_to_int(v['blockNumber']),
hash=v['hash'],
from_address=v['from'],
to_address=v['to'],
value=FVal(v['value']),
gas=FVal(v['gas']),
gas_price=gas_price,
gas_used=FVal(v['gasUsed']),
))
return result
def query_deposits_withdrawals(
self,
start_ts: Timestamp,
end_ts: Timestamp,
end_at_least_ts: Timestamp,
) -> List:
with self.lock:
cache = self.check_trades_cache(
start_ts,
end_at_least_ts,
special_name='deposits_withdrawals',
)
if cache is not None:
result = cache
else:
result = self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'),
)
result.extend(self.query_until_finished(
endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal'),
))
with self.lock:
self.update_trades_cache(
result,
start_ts,
end_ts,
special_name='deposits_withdrawals',
)
log.debug('Kraken deposit/withdrawals query result', num_results=len(result))
movements = list()
for movement in result:
movements.append(AssetMovement(
exchange='kraken',
category=movement['type'],
# Kraken timestamps have floating point
timestamp=convert_to_int(movement['time'], accept_only_exact=False),
asset=KRAKEN_TO_WORLD[movement['asset']],
amount=FVal(movement['amount']),
fee=FVal(movement['fee']),
))
return movements
def trade_from_kraken(kraken_trade):
"""Turn a kraken trade returned from kraken trade history to our common trade
history format"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position(currency_pair, 'second')
return Trade(
# Kraken timestamps have floating point ...
timestamp=convert_to_int(kraken_trade['time'], accept_only_exact=False),
pair=currency_pair,
type=kraken_trade['type'],
rate=FVal(kraken_trade['price']),
cost=FVal(kraken_trade['cost']),
cost_currency=quote_currency,
fee=FVal(kraken_trade['fee']),
fee_currency=quote_currency,
amount=FVal(kraken_trade['vol']),
location='kraken'
)
def trade_from_kraken(kraken_trade: Dict[str, Any]) -> Trade:
"""Turn a kraken trade returned from kraken trade history to our common trade
history format"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position(currency_pair, 'second')
# Kraken timestamps have floating point
timestamp = Timestamp(convert_to_int(kraken_trade['time'], accept_only_exact=False))
amount = FVal(kraken_trade['vol'])
cost = FVal(kraken_trade['cost'])
fee = FVal(kraken_trade['fee'])
order_type = trade_type_from_string(kraken_trade['type'])
rate = FVal(kraken_trade['price'])
if cost != amount * rate:
log.warning('cost ({cost}) != amount ({amount}) * rate ({rate}) for kraken trade')
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
location='kraken',
pair=currency_pair,
trade_type=order_type,
amount=amount,
rate=rate,
fee=fee,
fee_currency=quote_currency,
)
def trade_from_kraken(kraken_trade):
"""Turn a kraken trade returned from kraken trade history to our common trade
history format"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position(currency_pair, 'second')
# Kraken timestamps have floating point
timestamp = convert_to_int(kraken_trade['time'], accept_only_exact=False)
amount = FVal(kraken_trade['vol'])
cost = FVal(kraken_trade['cost'])
fee = FVal(kraken_trade['fee'])
order_type = kraken_trade['type']
rate = FVal(kraken_trade['price'])
log.debug(
'Processing kraken Trade',
sensitive_log=True,
timestamp=timestamp,
order_type=order_type,
kraken_pair=kraken_trade['pair'],
pair=currency_pair,
quote_currency=quote_currency,
amount=amount,
cost=cost,
fee=fee,
rate=rate,
)
return Trade(
timestamp=timestamp,
pair=currency_pair,
type=order_type,
rate=rate,
cost=cost,
cost_currency=quote_currency,
fee=fee,
fee_currency=quote_currency,
amount=amount,
location='kraken',
)
def query_historical_price(self, from_asset, to_asset, timestamp):
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
Args:
from_asset (str): The ticker symbol of the asset for which we want to know
the price.
to_asset (str): The ticker symbol of the asset against which we want to
know the price.
timestamp (int): The timestamp at which to query the price
"""
if from_asset == to_asset:
return 1
if from_asset not in self.cryptocompare_coin_list:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(from_asset, to_asset, timestamp)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
# print("loaded {}_{}".format(from_asset, to_asset))
assert timestamp > data[0]['time']
index = convert_to_int((timestamp - data[0]['time']) / 3600, accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index]['time'] - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1]['time'] - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index]['high'] is None or data[index]['low'] is None:
# If we get some None in the hourly set price to 0 so that we check daily price
price = FVal(0)
else:
price = FVal((data[index]['high'] + data[index]['low'])) / 2
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
# Just get the BTC price
asset_btc_price = self.query_historical_price(from_asset, 'BTC', timestamp)
btc_to_asset_price = self.query_historical_price('BTC', to_asset, timestamp)
price = asset_btc_price * btc_to_asset_price
else:
# attempt to get the daily price by timestamp
query_string = (
'https://min-api.cryptocompare.com/data/pricehistorical?'
'fsym={}&tsyms={}&ts={}'.format(
from_asset, to_asset, timestamp
))
if to_asset == 'BTC':
query_string += '&tryConversion=false'
resp = urlopen(Request(query_string))
resp = rlk_jsonloads(resp.read())
print('DAILY PRICE OF ASSET: "{}"'.format(resp))
if from_asset not in resp:
error_message = 'Failed to query cryptocompare for: "{}"'.format(query_string)
raise ValueError(error_message)
price = FVal(resp[from_asset][to_asset])
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset,
to_asset,
tsToDate(timestamp, formatstr='%d/%m/%Y, %H:%M:%S')
)
return price
def query_historical_price(self, from_asset, to_asset, timestamp):
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
Args:
from_asset (str): The ticker symbol of the asset for which we want to know
the price.
to_asset (str): The ticker symbol of the asset against which we want to
know the price.
timestamp (int): The timestamp at which to query the price
"""
log.debug(
'Querying historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
if from_asset == to_asset:
return 1
if from_asset not in self.cryptocompare_coin_list:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(from_asset, to_asset, timestamp)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
assert timestamp > data[0]['time']
index = convert_to_int((timestamp - data[0]['time']) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index]['time'] - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1]['time'] - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index]['high'] is None or data[index]['low'] is None:
# If we get some None in the hourly set price to 0 so that we check daily price
price = FVal(0)
else:
price = FVal((data[index]['high'] + data[index]['low'])) / 2
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
log.debug(
f"Coudn't find historical price from {from_asset} to "
f"{to_asset}. Comparing with BTC...", )
# Just get the BTC price
asset_btc_price = self.query_historical_price(
from_asset, 'BTC', timestamp)
btc_to_asset_price = self.query_historical_price(
'BTC', to_asset, timestamp)
price = asset_btc_price * btc_to_asset_price
else:
log.debug(
f"Coudn't find historical price from {from_asset} to "
f"{to_asset}. Attempting to get daily price...", )
# attempt to get the daily price by timestamp
cc_from_asset = world_to_cryptocompare(from_asset)
cc_to_asset = world_to_cryptocompare(to_asset)
log.debug(
'Querying cryptocompare for daily historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
)
query_string = (
'https://min-api.cryptocompare.com/data/pricehistorical?'
'fsym={}&tsyms={}&ts={}'.format(
cc_from_asset,
cc_to_asset,
timestamp,
))
if to_asset == 'BTC':
query_string += '&tryConversion=false'
resp = request_get(query_string)
if cc_from_asset not in resp:
error_message = 'Failed to query cryptocompare for: "{}"'.format(
query_string)
log.error(
'Cryptocompare query for daily historical price failed',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
error=error_message,
)
raise ValueError(error_message)
price = FVal(resp[cc_from_asset][cc_to_asset])
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset, to_asset,
tsToDate(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'))
log.debug('Got historical price',
from_asset=from_asset,
to_asset=to_asset,
timestamp=timestamp,
price=price)
return price
