def query_fiat_pair(base, quote, timestamp=None):
if base == quote:
return FVal(1.0)
if timestamp is None:
querystr = 'http://api.fixer.io/latest?base={}'.format(base)
else:
querystr = 'http://api.fixer.io/{}?base={}'.format(
tsToDate(timestamp, formatstr='%Y-%m-%d'), base
)
tries = 5
while True:
try:
resp = urlopen(Request(querystr))
resp = rlk_jsonloads(resp.read())
break
except:
if tries == 0:
raise ValueError('Timeout while trying to query euro price')
time.sleep(0.05)
tries -= 1
try:
return FVal(resp['rates'][quote])
except:
raise ValueError('Could not find a "{}" price for "{}"'.format(base, quote))
def first_connection(self):
if self.first_connection_made:
return
resp = self.query_private('TradeVolume',
req={
'pair': 'XETHXXBT',
'fee-info': True
})
with self.lock:
# Assuming all fees are the same for all pairs that we trade here,
# as long as they are normal orders on normal pairs.
self.taker_fee = FVal(resp['fees']['XETHXXBT']['fee'])
# Note from kraken api: If an asset pair is on a maker/taker fee
# schedule, the taker side is given in "fees" and maker side in
# "fees_maker". For pairs not on maker/taker, they will only be
# given in "fees".
if 'fees_maker' in resp:
self.maker_fee = FVal(resp['fees_maker']['XETHXXBT']['fee'])
else:
self.maker_fee = self.taker_fee
self.tradeable_pairs = self.query_public('AssetPairs')
self.first_connection_made = True
# Also need to do at least a single pass of the main logic for the ticker
self.main_logic()
def get_multieth_balance(self, accounts):
"""Returns a dict with keys being accounts and balances in ETH"""
balances = {}
if not self.connected:
# TODO: accounts.length should be less than 20. If more we gotta do
# multiple calls
eth_resp = urlopen(
Request(
'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s'
% ','.join(accounts)))
eth_resp = rlk_jsonloads(eth_resp.read())
if eth_resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for accounts balance')
eth_accounts = eth_resp['result']
for account_entry in eth_accounts:
amount = FVal(account_entry['balance'])
balances[account_entry['account']] = from_wei(amount)
else:
for account in accounts:
amount = FVal(self.web3.eth.getBalance(account))
balances[account] = from_wei(amount)
return balances
def query_deposits_withdrawals(self, start_ts, end_ts, end_at_least_ts):
with self.lock:
cache = self.check_trades_cache(
start_ts, end_at_least_ts, special_name='deposits_withdrawals')
if cache is None:
result = self.returnDepositsWithdrawals(start_ts, end_ts)
with self.lock:
self.update_trades_cache(result,
start_ts,
end_ts,
special_name='deposits_withdrawals')
else:
result = cache
movements = list()
for withdrawal in result['withdrawals']:
movements.append(
AssetMovement(exchange='poloniex',
category='withdrawal',
timestamp=withdrawal['timestamp'],
asset=withdrawal['currency'],
amount=FVal(withdrawal['amount']),
fee=FVal(withdrawal['fee'])))
for deposit in result['deposits']:
movements.append(
AssetMovement(exchange='poloniex',
category='deposit',
timestamp=deposit['timestamp'],
asset=deposit['currency'],
amount=FVal(deposit['amount']),
fee=0))
return movements
def trade_from_binance(binance_trade):
"""Turn a binance trade returned from trade history to our common trade
history format"""
amount = FVal(binance_trade['qty'])
rate = FVal(binance_trade['price'])
pair = binance_pair_to_world(binance_trade['symbol'])
base_asset, quote_asset = pair_get_assets(pair)
if binance_trade['isBuyer']:
order_type = 'buy'
# e.g. in RDNETH we buy RDN by paying ETH
cost_currency = quote_asset
else:
order_type = 'sell'
# e.g. in RDNETH we sell RDN to obtain ETH
cost_currency = base_asset
fee_currency = binance_trade['commissionAsset']
fee = FVal(binance_trade['commission'])
cost = rate * amount
return Trade(
timestamp=binance_trade['time'],
pair=pair,
type=order_type,
rate=rate,
cost=cost,
cost_currency=cost_currency,
fee=fee,
fee_currency=fee_currency,
amount=amount,
location='binance'
)
def find_usd_price(self, asset, asset_btc_price=None):
if self.kraken and self.kraken.first_connection_made and asset_btc_price is not None:
return self.query_kraken_for_price(asset, asset_btc_price)
# Adjust some ETH tokens to how cryptocompare knows them
if asset == 'RDN':
asset = 'RDN*' # temporary
if asset == 'DATAcoin':
asset = 'DATA'
resp = retry_calls(
5,
'find_usd_price',
'urllib2.urlopen',
urlopen,
Request(
u'https://min-api.cryptocompare.com/data/price?fsym={}&tsyms=USD'.format(
asset
))
)
resp = rlk_jsonloads(resp.read())
# If there is an error in the response skip this token
if 'USD' not in resp:
if resp['Response'] == 'Error':
print('Could not query USD price for {}. Error: "{}"'.format(
asset,
resp['Message']),
)
else:
print('Could not query USD price for {}'.format(asset))
return FVal(0)
return FVal(resp['USD'])
def test_decoding():
strdata = """
{"a": 3.14, "b":5, "c": "foo", "d": "5.42323143", "e": { "u1": "3.221"}, "f": [2.1, "boo", 3, "4.2324"]}"""
data = rlk_jsonloads(strdata)
assert isinstance(data['a'], FVal)
assert data['a'] == FVal('3.14')
assert isinstance(data['b'], int)
assert data['b'] == 5
assert isinstance(data['c'], (str, bytes))
assert data['c'] == 'foo'
assert isinstance(data['d'], FVal)
assert data['d'] == FVal('5.42323143')
assert isinstance(data['e']['u1'], FVal)
assert data['e']['u1'] == FVal('3.221')
assert isinstance(data['f'][0], FVal)
assert data['f'][0] == FVal('2.1')
assert isinstance(data['f'][1], (str, bytes))
assert data['f'][1] == "boo"
assert isinstance(data['f'][2], int)
assert data['f'][2] == 3
assert isinstance(data['f'][3], FVal)
assert data['f'][3] == FVal('4.2324')
def trade_from_bittrex(bittrex_trade):
"""Turn a bittrex trade returned from bittrex trade history to our common trade
history format"""
amount = FVal(bittrex_trade['Quantity']) - FVal(
bittrex_trade['QuantityRemaining'])
rate = FVal(bittrex_trade['PricePerUnit'])
order_type = bittrex_trade['OrderType']
bittrex_price = FVal(bittrex_trade['Price'])
bittrex_commission = FVal(bittrex_trade['Commission'])
pair = bittrex_pair_to_world(bittrex_trade['Exchange'])
base_currency = get_pair_position(pair, 'first')
if order_type == 'LIMIT_BUY':
order_type = 'buy'
cost = bittrex_price + bittrex_commission
fee = bittrex_commission
elif order_type == 'LIMIT_SEL':
order_type = 'sell'
cost = bittrex_price - bittrex_commission
fee = bittrex_commission
else:
raise ValueError(
'Got unexpected order type "{}" for bittrex trade'.format(
order_type))
return Trade(timestamp=bittrex_trade['TimeStamp'],
pair=pair,
type=order_type,
rate=rate,
cost=cost,
cost_currency=base_currency,
fee=fee,
fee_currency=base_currency,
amount=amount,
location='bittrex')
def transactions_from_dictlist(given_transactions, start_ts, end_ts):
""" Gets a list of transaction, most probably read from the json files and
a time period. Returns it as a list of the transaction tuples that are inside the time period
"""
returned_transactions = list()
for given_tx in given_transactions:
if given_tx['timestamp'] < start_ts:
continue
if given_tx['timestamp'] > end_ts:
break
returned_transactions.append(
EthereumTransaction(
timestamp=convert_to_int(given_tx['timestamp']),
block_number=convert_to_int(given_tx['block_number']),
hash=given_tx['hash'],
from_address=given_tx['from_address'],
to_address=given_tx['to_address'],
value=FVal(given_tx['value']),
gas=FVal(given_tx['gas']),
gas_price=FVal(given_tx['gas_price']),
gas_used=FVal(given_tx['gas_used']),
))
return returned_transactions
def test_representation():
a = FVal(2.01)
b = FVal('2.01')
assert a == b
a = FVal(2.00)
b = FVal('2.0')
c = FVal(2)
assert a == b
assert b == c
def first_connection(self):
if self.first_connection_made:
return
fees_resp = self.returnFeeInfo()
with self.lock:
self.maker_fee = FVal(fees_resp['makerFee'])
self.taker_fee = FVal(fees_resp['takerFee'])
self.first_connection_made = True
# Also need to do at least a single pass of the market watcher for the ticker
self.market_watcher()
def parseLoanCSV(self, path):
self.lending_history = []
with open(path, 'rb') as csvfile:
history = csv.reader(csvfile, delimiter=',', quotechar='|')
next(history) # skip header row
for row in history:
self.lending_history.append({
'currency': row[0],
'earned': FVal(row[6]),
'amount': FVal(row[2]),
'opened': createTimeStamp(row[7]),
'closed': createTimeStamp(row[8])
})
return self.lending_history
def test_arithmetic_with_int():
a = FVal(5.21)
assert a - 2 == FVal('3.21')
assert a + 2 == FVal('7.21')
assert a * 2 == FVal('10.42')
assert a / 2 == FVal('2.605')
assert a**3 == FVal('141.420761')
# and now the reverse operations
assert 2 + a == FVal('7.21')
assert 2 - a == FVal('-3.21')
assert 2 * a == FVal('10.42')
assert 2 / a == FVal('0.3838771593090211132437619962')
def query_balances(self):
resp = self.api_query('returnCompleteBalances', {"account": "all"})
balances = dict()
for currency, v in resp.items():
available = FVal(v['available'])
on_orders = FVal(v['onOrders'])
if (available != FVal(0) or on_orders != FVal(0)):
entry = {}
entry['amount'] = available + on_orders
usd_price = self.inquirer.find_usd_price(asset=currency,
asset_btc_price=None)
usd_value = entry['amount'] * usd_price
entry['usd_value'] = usd_value
balances[currency] = entry
return balances
def query_ethereum_balances(self):
if 'ETH' not in self.accounts:
return
eth_accounts = self.accounts['ETH']
eth_usd_price = self.inquirer.find_usd_price('ETH')
balances = self.ethchain.get_multieth_balance(eth_accounts)
eth_total = FVal(0)
eth_balances = {}
for account, balance in balances.items():
eth_total += balance
eth_balances[account] = {
'ETH': balance,
'usd_value': balance * eth_usd_price
}
self.totals['ETH'] = {
'amount': eth_total,
'usd_value': eth_total * eth_usd_price
}
self.balances[
'ETH'] = eth_balances # but they are not complete until token query
# And now for tokens
self.query_ethereum_tokens(self.owned_eth_tokens, eth_balances)
def find_fiat_price(self, asset):
"""Find USD/EUR price of asset. The asset should be in the kraken style.
e.g.: XICN. Save both prices in the kraken object and then return the
USD price.
"""
if asset == 'XXBT':
return self.usdprice['BTC']
# TODO: This is pretty ugly. Find a better way to check out kraken pairs
# without this ugliness.
pair = asset + 'XXBT'
pair2 = asset + 'XBT'
if pair2 in self.tradeable_pairs:
pair = pair2
if pair not in self.tradeable_pairs:
raise ValueError(
'Could not find a BTC tradeable pair in kraken for "{}"'.
format(asset))
btc_price = FVal(self.ticker[pair]['c'][0])
common_name = KRAKEN_TO_WORLD[asset]
with self.lock:
self.usdprice[common_name] = btc_price * self.usdprice['BTC']
self.eurprice[common_name] = btc_price * self.eurprice['BTC']
return self.usdprice[common_name]
def query_balances(self):
self.markets = self.api_query('getmarketsummaries')
resp = self.api_query('getbalances')
returned_balances = dict()
for entry in resp:
currency = entry['Currency']
usd_price = self.inquirer.find_usd_price(
asset=currency, asset_btc_price=self.get_btc_price(currency))
balance = dict()
balance['amount'] = FVal(entry['Balance'])
balance['usd_value'] = FVal(balance['amount']) * usd_price
returned_balances[currency] = balance
return returned_balances
def query_balances(self):
account_data = self.api_query('account')
returned_balances = dict()
for entry in account_data['balances']:
amount = entry['free'] + entry['locked']
if amount == FVal(0):
continue
currency = entry['asset']
usd_price = self.inquirer.find_usd_price(currency)
balance = dict()
balance['amount'] = amount
balance['usd_value'] = FVal(amount * usd_price)
returned_balances[currency] = balance
return returned_balances
def query_ethereum_tokens(self, tokens, eth_balances):
token_balances = {}
token_usd_price = {}
for token in tokens:
usd_price = self.inquirer.find_usd_price(token)
if usd_price == 0:
# skip tokens that have no price
continue
token_usd_price[token] = usd_price
token_balances[token] = self.ethchain.get_multitoken_balance(
token,
self.all_eth_tokens[token]['address'],
self.all_eth_tokens[token]['decimal'],
self.accounts['ETH'],
)
for token, token_accounts in token_balances.items():
token_total = FVal(0)
for account, balance in token_accounts.items():
token_total += balance
usd_value = balance * token_usd_price[token]
eth_balances[account][token] = balance
eth_balances[account]['usd_value'] = eth_balances[account][
'usd_value'] + usd_value
self.totals[token] = {
'amount': token_total,
'usd_value': token_total * token_usd_price[token]
}
self.balances['ETH'] = eth_balances
def parseLoanCSV(self):
# the default filename, and should be (if at all) inside the data directory
path = os.path.join(self.data_dir, "lendingHistory.csv")
lending_history = list()
with open(path, 'rb') as csvfile:
history = csv.reader(csvfile, delimiter=',', quotechar='|')
next(history) # skip header row
for row in history:
lending_history.append({
'currency': row[0],
'earned': FVal(row[6]),
'amount': FVal(row[2]),
'fee': FVal(row[5]),
'open': row[7],
'close': row[8]
})
return lending_history
def test_comparison():
a = FVal('1.348938409')
b = FVal('0.123432434')
c = FVal('1.348938410')
d = FVal('1.348938409')
assert a > b
assert a >= b
assert b < a
assert b <= a
assert c > a
assert c >= a
assert a < c
assert a <= c
assert a == d
assert a <= d
assert a >= d
def trade_from_kraken(kraken_trade):
"""Turn a kraken trade returned from kraken trade history to our common trade
history format"""
currency_pair = kraken_to_world_pair(kraken_trade['pair'])
quote_currency = get_pair_position(currency_pair, 'second')
return Trade(
# Kraken timestamps have floating point ...
timestamp=convert_to_int(kraken_trade['time'],
accept_only_exact=False),
pair=currency_pair,
type=kraken_trade['type'],
rate=FVal(kraken_trade['price']),
cost=FVal(kraken_trade['cost']),
cost_currency=quote_currency,
fee=FVal(kraken_trade['fee']),
fee_currency=quote_currency,
amount=FVal(kraken_trade['vol']),
location='kraken')
def rkl_decode_value(val):
if isinstance(val, dict):
new_val = dict()
for k, v in val.items():
new_val[k] = rkl_decode_value(v)
return new_val
elif isinstance(val, list):
return [rkl_decode_value(x) for x in val]
elif isinstance(val, float):
return FVal(val)
elif isinstance(val, (bytes, str)):
try:
val = float(val)
return FVal(val)
except:
pass
return val
def main_logic(self):
if not self.first_connection_made:
return
self.ticker = self.query_public(
'Ticker', req={'pair': ','.join(self.tradeable_pairs.keys())})
self.eurprice['BTC'] = FVal(self.ticker['XXBTZEUR']['c'][0])
self.usdprice['BTC'] = FVal(self.ticker['XXBTZUSD']['c'][0])
self.eurprice['ETH'] = FVal(self.ticker['XETHZEUR']['c'][0])
self.usdprice['ETH'] = FVal(self.ticker['XETHZUSD']['c'][0])
self.eurprice['REP'] = FVal(self.ticker['XREPZEUR']['c'][0])
self.eurprice['XMR'] = FVal(self.ticker['XXMRZEUR']['c'][0])
self.usdprice['XMR'] = FVal(self.ticker['XXMRZUSD']['c'][0])
self.eurprice['ETC'] = FVal(self.ticker['XETCZEUR']['c'][0])
self.usdprice['ETC'] = FVal(self.ticker['XETCZUSD']['c'][0])
def query_txlist(address, internal, from_block=None, to_block=None):
result = list()
if internal:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address))
else:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address))
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = urlopen(Request(reqstring))
resp = rlk_jsonloads(resp.read())
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp))
for v in resp['result']:
# internal tx list contains no gasprice
gas_price = -1 if internal else FVal(v['gasPrice'])
result.append(
EthereumTransaction(
timestamp=convert_to_int(v['timeStamp']),
block_number=convert_to_int(v['blockNumber']),
hash=v['hash'],
from_address=v['from'],
to_address=v['to'],
value=FVal(v['value']),
gas=FVal(v['gas']),
gas_price=gas_price,
gas_used=FVal(v['gasUsed']),
))
return result
def query_fiat_balances(self):
result = {}
balances = self.data.get_fiat_balances()
for currency, amount in balances.items():
amount = FVal(amount)
usd_rate = query_fiat_pair(currency, 'USD')
result[currency] = {
'amount': amount,
'usd_value': amount * usd_rate
}
return result
def asset_movements_from_dictlist(given_data, start_ts, end_ts):
""" Gets a list of dict asset movements, most probably read from the json files and
a time period. Returns it as a list of the AssetMovement tuples that are inside the time period
"""
returned_movements = list()
for movement in given_data:
if movement['timestamp'] < start_ts:
continue
if movement['timestamp'] > end_ts:
break
returned_movements.append(
AssetMovement(
exchange=movement['exchange'],
category=movement['category'],
timestamp=movement['timestamp'],
asset=movement['asset'],
amount=FVal(movement['amount']),
fee=FVal(movement['fee']),
))
return returned_movements
def trade_from_poloniex(poloniex_trade, pair):
"""Turn a poloniex trade returned from poloniex trade history to our common trade
history format"""
trade_type = poloniex_trade['type']
amount = FVal(poloniex_trade['amount'])
rate = FVal(poloniex_trade['rate'])
perc_fee = FVal(poloniex_trade['fee'])
base_currency = get_pair_position(pair, 'first')
quote_currency = get_pair_position(pair, 'second')
if trade_type == 'buy':
cost = rate * amount
cost_currency = base_currency
fee = amount * perc_fee
fee_currency = quote_currency
elif trade_type == 'sell':
cost = amount * rate
cost_currency = base_currency
fee = cost * perc_fee
fee_currency = base_currency
else:
raise ValueError(
'Got unexpected trade type "{}" for poloniex trade'.format(
trade_type))
if poloniex_trade['category'] == 'settlement':
trade_type = "settlement_%s" % trade_type
return Trade(timestamp=createTimeStamp(poloniex_trade['date'],
formatstr="%Y-%m-%d %H:%M:%S"),
pair=pair,
type=trade_type,
rate=rate,
cost=cost,
cost_currency=cost_currency,
fee=fee,
fee_currency=fee_currency,
amount=amount,
location='poloniex')
def query_deposits_withdrawals(self, start_ts, end_ts, end_at_least_ts):
with self.lock:
cache = self.check_trades_cache(
start_ts, end_at_least_ts, special_name='deposits_withdrawals')
if cache is not None:
result = cache
else:
result = self.query_until_finished(endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='deposit'))
result.extend(
self.query_until_finished(endpoint='Ledgers',
keyname='ledger',
start_ts=start_ts,
end_ts=end_ts,
extra_dict=dict(type='withdrawal')))
with self.lock:
self.update_trades_cache(result,
start_ts,
end_ts,
special_name='deposits_withdrawals')
movements = list()
for movement in result:
movements.append(
AssetMovement(
exchange='kraken',
category=movement['type'],
# Kraken timestamps have floating point
timestamp=convert_to_int(movement['time'],
accept_only_exact=False),
asset=KRAKEN_TO_WORLD[movement['asset']],
amount=FVal(movement['amount']),
fee=FVal(movement['fee'])))
return movements
def process_polo_loans(data, start_ts, end_ts):
new_data = list()
for loan in reversed(data):
close_time = createTimeStamp(loan['close'],
formatstr="%Y-%m-%d %H:%M:%S")
open_time = createTimeStamp(loan['open'],
formatstr="%Y-%m-%d %H:%M:%S")
if open_time < start_ts:
continue
if close_time > end_ts:
break
new_data.append({
'open_time': open_time,
'close_time': close_time,
'currency': loan['currency'],
'fee': FVal(loan['fee']),
'earned': FVal(loan['earned']),
'amount_lent': FVal(loan['amount']),
})
new_data.sort(key=lambda loan: loan['open_time'])
return new_data