def _api_query(self, command, req={}):
if command == "returnTicker" or command == "return24Volume":
ret = self.session.get(self.public_uri + command)
elif (command == "returnOrderBook"):
ret = self.session.get(self.public_uri + command +
'¤cyPair=' + str(req['currencyPair']))
elif (command == "returnMarketTradeHistory"):
ret = self.session.get(self.public_uri + 'returnTradeHistory' +
'¤cyPair=' + str(req['currencyPair']))
elif (command == "returnLoanOrders"):
ret = self.session.get(self.public_uri + 'returnLoanOrders' +
'¤cy=' + str(req['currency']))
else:
req['command'] = command
with self.lock:
# Protect this region with a lock since poloniex will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
req['nonce'] = int(time.time() * 1000)
post_data = str.encode(urlencode(req))
sign = hmac.new(self.secret, post_data,
hashlib.sha512).hexdigest()
self.session.headers.update({'Sign': sign})
ret = self.session.post('https://poloniex.com/tradingApi', req)
result = rlk_jsonloads(ret.text)
return self.post_process(result)
return rlk_jsonloads(ret.text)
def api_query(self, method, options=None):
"""
Queries Bittrex with given method and options
"""
if not options:
options = {}
nonce = str(int(time.time() * 1000))
method_type = 'public'
if method in BITTREX_MARKET_METHODS:
method_type = 'market'
elif method in BITTREX_ACCOUNT_METHODS:
method_type = 'account'
request_url = self.uri + method_type + '/' + method + '?'
if method_type != 'public':
request_url += 'apikey=' + self.api_key.decode(
) + "&nonce=" + nonce + '&'
request_url += urlencode(options)
signature = hmac.new(self.secret, request_url.encode(),
hashlib.sha512).hexdigest()
self.session.headers.update({'apisign': signature})
response = self.session.get(request_url)
json_ret = rlk_jsonloads(response.text)
if json_ret['success'] is not True:
raise ValueError(json_ret['message'])
return json_ret['result']
def get_multieth_balance(self, accounts):
"""Returns a dict with keys being accounts and balances in ETH"""
balances = {}
if not self.connected:
# TODO: accounts.length should be less than 20. If more we gotta do
# multiple calls
eth_resp = urlopen(
Request(
'https://api.etherscan.io/api?module=account&action=balancemulti&address=%s'
% ','.join(accounts)))
eth_resp = rlk_jsonloads(eth_resp.read())
if eth_resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for accounts balance')
eth_accounts = eth_resp['result']
for account_entry in eth_accounts:
amount = FVal(account_entry['balance'])
balances[account_entry['account']] = from_wei(amount)
else:
for account in accounts:
amount = FVal(self.web3.eth.getBalance(account))
balances[account] = from_wei(amount)
return balances
def find_usd_price(self, asset, asset_btc_price=None):
if self.kraken and self.kraken.first_connection_made and asset_btc_price is not None:
return self.query_kraken_for_price(asset, asset_btc_price)
# Adjust some ETH tokens to how cryptocompare knows them
if asset == 'RDN':
asset = 'RDN*' # temporary
if asset == 'DATAcoin':
asset = 'DATA'
resp = retry_calls(
5,
'find_usd_price',
'urllib2.urlopen',
urlopen,
Request(
u'https://min-api.cryptocompare.com/data/price?fsym={}&tsyms=USD'.format(
asset
))
)
resp = rlk_jsonloads(resp.read())
# If there is an error in the response skip this token
if 'USD' not in resp:
if resp['Response'] == 'Error':
print('Could not query USD price for {}. Error: "{}"'.format(
asset,
resp['Message']),
)
else:
print('Could not query USD price for {}'.format(asset))
return FVal(0)
return FVal(resp['USD'])
def test_decoding():
strdata = """
{"a": 3.14, "b":5, "c": "foo", "d": "5.42323143", "e": { "u1": "3.221"}, "f": [2.1, "boo", 3, "4.2324"]}"""
data = rlk_jsonloads(strdata)
assert isinstance(data['a'], FVal)
assert data['a'] == FVal('3.14')
assert isinstance(data['b'], int)
assert data['b'] == 5
assert isinstance(data['c'], (str, bytes))
assert data['c'] == 'foo'
assert isinstance(data['d'], FVal)
assert data['d'] == FVal('5.42323143')
assert isinstance(data['e']['u1'], FVal)
assert data['e']['u1'] == FVal('3.221')
assert isinstance(data['f'][0], FVal)
assert data['f'][0] == FVal('2.1')
assert isinstance(data['f'][1], (str, bytes))
assert data['f'][1] == "boo"
assert isinstance(data['f'][2], int)
assert data['f'][2] == 3
assert isinstance(data['f'][3], FVal)
assert data['f'][3] == FVal('4.2324')
def check_and_get_response(self, response, method):
if response.status_code in (520, 525, 504):
raise RecoverableRequestError('kraken',
'Usual kraken 5xx shenanigans')
elif response.status_code != 200:
raise ValueError(
'Kraken API request {} for {} failed with HTTP status '
'code: {}'.format(
response.url,
method,
response.status_code,
))
result = rlk_jsonloads(response.text)
if result['error']:
if isinstance(result['error'], list):
error = result['error'][0]
else:
error = result['error']
if 'Rate limit exceeded' in error:
raise RecoverableRequestError('kraken',
'Rate limited exceeded')
else:
raise ValueError(error)
return result['result']
def __init__(self, data_directory):
self.data_directory = data_directory
try:
with open(os.path.join(self.data_directory, 'settings.json')) as f:
self.settings = rlk_jsonloads(f.read())
except JSONDecodeError as e:
logger.critical('settings.json file could not be decoded and is corrupt: {}'.format(e))
self.settings = empty_settings
except FileNotFoundError:
self.settings = empty_settings
self.db = None
self.eth_tokens = []
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(dir_path, 'data', 'eth_tokens.json'), 'r') as f:
self.eth_tokens = rlk_jsonloads(f.read())
def do_read_manual_margin_positions(data_directory):
manual_margin_path = os.path.join(data_directory, MANUAL_MARGINS_LOGFILE)
if os.path.isfile(manual_margin_path):
with open(manual_margin_path, 'r') as f:
margin_data = rlk_jsonloads(f.read())
else:
margin_data = []
logger.error('Could not find manual margins log file at {}'.format(
manual_margin_path))
return margin_data
def api_query(self, method, options=None):
if not options:
options = {}
with self.lock:
# Protect this region with a lock since binance will reject
# non-increasing nonces. So if two greenlets come in here at
# the same time one of them will fail
if method in V3_ENDPOINTS:
api_version = 3
# Recommended recvWindows is 5000 but we get timeouts with it
options['recvWindow'] = 10000
options['timestamp'] = str(int(time.time() * 1000))
signature = hmac.new(
self.secret,
urlencode(options).encode('utf-8'),
hashlib.sha256
).hexdigest()
options['signature'] = signature
elif method in V1_ENDPOINTS:
api_version = 1
else:
raise ValueError('Unexpected binance api method {}'.format(method))
request_url = self.uri + 'v' + str(api_version) + '/' + method + '?'
request_url += urlencode(options)
response = self.session.get(request_url)
if response.status_code != 200:
result = rlk_jsonloads(response.text)
raise ValueError(
'Binance API request {} for {} failed with HTTP status '
'code: {}, error code: {} and error message: {}'.format(
response.url,
method,
response.status_code,
result['code'],
result['msg'],
))
json_ret = rlk_jsonloads(response.text)
return json_ret
def get_eth_balance(self, account):
if not self.connected:
eth_resp = urlopen(
Request(
'https://api.etherscan.io/api?module=account&action=balance&address=%s'
% account))
eth_resp = rlk_jsonloads(eth_resp.read())
if eth_resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for accounts balance')
amount = FVal(eth_resp['result'])
return from_wei(amount)
else:
return from_wei(self.web3.eth.getBalance(account))
def process_response(self, response):
result_or_error = ''
success = False
if response.status_code not in HANDLABLE_STATUS_CODES:
result_or_error = (
'Unexpected status response({}) from rotkehlchen server'.
format(response.status_code))
else:
result_or_error = rlk_jsonloads(response.text)
if 'error' in result_or_error:
result_or_error = result_or_error['error']
else:
success = True
return success, result_or_error
def check_trades_cache(self, start_ts, end_ts, special_name=None):
trades_file = self._get_cachefile_name(special_name)
trades = dict()
if os.path.isfile(trades_file):
with open(trades_file, 'r') as f:
try:
trades = rlk_jsonloads(f.read())
except:
pass
# no need to query again
if data_up_todate(trades, start_ts, end_ts):
return trades['data']
return None
def __init__(self, ethrpc_port):
self.connected = True
# Note that you should create only one RPCProvider per
# process, as it recycles underlying TCP/IP network connections between
# your process and Ethereum node
self.web3 = Web3(
HTTPProvider('http://localhost:{}'.format(ethrpc_port)))
try:
self.web3.eth.blockNumber
dir_path = os.path.dirname(os.path.realpath(__file__))
with open(os.path.join(dir_path, 'data', 'token_abi.json'),
'r') as f:
self.token_abi = rlk_jsonloads(f.read())
except ConnectionError:
logger.warn(
'Could not connect to a local ethereum node. Will use etherscan only'
)
self.connected = False
def query_txlist(address, internal, from_block=None, to_block=None):
result = list()
if internal:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlistinternal&address={}'.format(address))
else:
reqstring = ('https://api.etherscan.io/api?module=account&action='
'txlist&address={}'.format(address))
if from_block:
reqstring += '&startblock={}'.format(from_block)
if to_block:
reqstring += '&endblock={}'.format(to_block)
resp = urlopen(Request(reqstring))
resp = rlk_jsonloads(resp.read())
if 'status' not in resp or convert_to_int(resp['status']) != 1:
status = convert_to_int(resp['status'])
if status == 0 and resp['message'] == 'No transactions found':
return list()
# else unknown error
raise ValueError(
'Failed to query txlist from etherscan with query: {} . '
'Response was: {}'.format(reqstring, resp))
for v in resp['result']:
# internal tx list contains no gasprice
gas_price = -1 if internal else FVal(v['gasPrice'])
result.append(
EthereumTransaction(
timestamp=convert_to_int(v['timeStamp']),
block_number=convert_to_int(v['blockNumber']),
hash=v['hash'],
from_address=v['from'],
to_address=v['to'],
value=FVal(v['value']),
gas=FVal(v['gas']),
gas_price=gas_price,
gas_used=FVal(v['gasUsed']),
))
return result
def get_multitoken_balance(self, token_symbol, token_address,
token_decimals, accounts):
"""Return a dictionary with keys being accounts and value balances of token
Balance value is normalized through the token decimals.
"""
balances = {}
if self.connected:
token_contract = self.web3.eth.contract(address=token_address,
abi=self.token_abi)
for account in accounts:
token_amount = FVal(token_contract.call().balanceOf(account))
if token_amount != 0:
balances[account] = token_amount / (FVal(10)**
FVal(token_decimals))
else:
for account in accounts:
print('Checking token {} for account {}'.format(
token_symbol, account))
resp = urlopen(
Request(
'https://api.etherscan.io/api?module=account&action='
'tokenbalance&contractaddress={}&address={}'.format(
token_address,
account,
)))
resp = rlk_jsonloads(resp.read())
if resp['status'] != 1:
raise ValueError(
'Failed to query etherscan for {} token balance of {}'.
format(
token_symbol,
account,
))
token_amount = FVal(resp['result'])
if token_amount != 0:
balances[account] = token_amount / (FVal(10)**
FVal(token_decimals))
return balances
def __init__(self, data_directory, history_date_start=DEFAULT_START_DATE):
self.data_directory = data_directory
# get the start date for historical data
self.historical_data_start = createTimeStamp(history_date_start,
formatstr="%d/%m/%Y")
self.price_history = dict()
# TODO: Check if historical data is after the requested start date
# Check the data folder and load any cached history
prefix = os.path.join(self.data_directory, 'price_history_')
regex = re.compile(prefix + '(.*)\.json')
files_list = glob.glob(prefix + '*.json')
for file_ in files_list:
match = regex.match(file_)
assert match
cache_key = match.group(1)
with open(file_, 'rb') as f:
data = rlk_jsonloads(f.read())
self.price_history[cache_key] = data
# Get coin list of crypto compare
invalidate_cache = True
coinlist_cache_path = os.path.join(self.data_directory,
'cryptocompare_coinlist.json')
if os.path.isfile(coinlist_cache_path):
with open(coinlist_cache_path, 'rb') as f:
try:
data = rlk_jsonloads(f.read())
now = ts_now()
invalidate_cache = False
# If we got a cache and its' over a month old then requery cryptocompare
if data['time'] < now and now - data['time'] > 2629800:
invalidate_cache = True
data = data['data']
except JSONDecodeError:
invalidate_cache = True
if invalidate_cache:
query_string = 'https://www.cryptocompare.com/api/data/coinlist/'
resp = urlopen(Request(query_string))
resp = rlk_jsonloads(resp.read())
if 'Response' not in resp or resp['Response'] != 'Success':
error_message = 'Failed to query cryptocompare for: "{}"'.format(
query_string)
if 'Message' in resp:
error_message += ". Error: {}".format(resp['Message'])
raise ValueError(error_message)
data = resp['Data']
# Also save the cache
with open(coinlist_cache_path, 'w') as f:
write_data = {'time': ts_now(), 'data': data}
f.write(rlk_jsondumps(write_data))
else:
# in any case take the data
data = data['data']
self.cryptocompare_coin_list = data
# For some reason even though price for the following assets is returned
# it's not in the coinlist so let's add them here.
self.cryptocompare_coin_list['DAO'] = object()
self.cryptocompare_coin_list['USDT'] = object()
def get_historical_data(self, from_asset, to_asset, timestamp):
"""Get historical price data from cryptocompare"""
if from_asset not in self.cryptocompare_coin_list:
raise ValueError('Attempted to query historical price data for '
'unknown asset "{}"'.format(from_asset))
if to_asset not in self.cryptocompare_coin_list and to_asset not in FIAT_CURRENCIES:
raise ValueError('Attempted to query historical price data for '
'unknown asset "{}"'.format(to_asset))
cache_key = from_asset + '_' + to_asset
if cache_key in self.price_history and self.price_history[cache_key][
'end_time'] > timestamp:
return self.price_history[cache_key]['data']
now_ts = int(time.time())
cryptocompare_hourquerylimit = 2000
calculated_history = list()
end_date = self.historical_data_start
while True:
pr_end_date = end_date
end_date = end_date + (cryptocompare_hourquerylimit) * 3600
query_string = ('https://min-api.cryptocompare.com/data/histohour?'
'fsym={}&tsym={}&limit={}&toTs={}'.format(
from_asset, to_asset,
cryptocompare_hourquerylimit, end_date))
resp = urlopen(Request(query_string))
resp = rlk_jsonloads(resp.read())
if 'Response' not in resp or resp['Response'] != 'Success':
error_message = 'Failed to query cryptocompare for: "{}"'.format(
query_string)
if 'Message' in resp:
error_message += ". Error: {}".format(resp['Message'])
raise ValueError(error_message)
if pr_end_date != resp['TimeFrom']:
# If we get more than we needed, since we are close to the now_ts
# then skip all the already included entries
diff = pr_end_date - resp['TimeFrom']
if resp['Data'][diff / 3600]['time'] != pr_end_date:
raise ValueError(
'Expected to find the previous date timestamp during '
'historical data fetching')
# just add only the part from the previous timestamp and on
resp['Data'] = resp['Data'][diff / 3600:]
if end_date < now_ts and resp['TimeTo'] != end_date:
raise ValueError('End dates no match')
# If last time slot and first new are the same, skip the first new slot
last_entry_equal_to_first = (len(calculated_history) != 0
and calculated_history[-1]['time']
== resp['Data'][0]['time'])
if last_entry_equal_to_first:
resp['Data'] = resp['Data'][1:]
calculated_history += resp['Data']
if end_date >= now_ts:
break
# Let's always check for data sanity for the hourly prices.
assert check_hourly_data_sanity(calculated_history, from_asset,
to_asset)
self.price_history[cache_key] = {
'data': calculated_history,
'start_time': self.historical_data_start,
'end_time': now_ts
}
# and now since we actually queried the data let's also save them locally
write_history_data_in_file(
calculated_history,
os.path.join(self.data_directory,
'price_history_' + cache_key + '.json'),
self.historical_data_start, now_ts)
return calculated_history
def query_historical_price(self, from_asset, to_asset, timestamp):
"""
Query the historical price on `timestamp` for `from_asset` in `to_asset`.
So how much `to_asset` does 1 unit of `from_asset` cost.
Args:
from_asset (str): The ticker symbol of the asset for which we want to know
the price.
to_asset (str): The ticker symbol of the asset against which we want to
know the price.
timestamp (int): The timestamp at which to query the price
"""
if from_asset == to_asset:
return 1
if from_asset not in self.cryptocompare_coin_list:
raise PriceQueryUnknownFromAsset(from_asset)
data = self.get_historical_data(from_asset, to_asset, timestamp)
# all data are sorted and timestamps are always increasing by 1 hour
# find the closest entry to the provided timestamp
# print("loaded {}_{}".format(from_asset, to_asset))
assert timestamp > data[0]['time']
index = convert_to_int((timestamp - data[0]['time']) / 3600,
accept_only_exact=False)
# print("timestamp: {} index: {} data_length: {}".format(timestamp, index, len(data)))
diff = abs(data[index]['time'] - timestamp)
if index + 1 <= len(data) - 1:
diff_p1 = abs(data[index + 1]['time'] - timestamp)
if diff_p1 < diff:
index = index + 1
if data[index]['high'] is None or data[index]['low'] is None:
# If we get some None in the hourly set price to 0 so that we check daily price
price = FVal(0)
else:
price = FVal((data[index]['high'] + data[index]['low'])) / 2
if price == 0:
if from_asset != 'BTC' and to_asset != 'BTC':
# Just get the BTC price
asset_btc_price = self.query_historical_price(
from_asset, 'BTC', timestamp)
btc_to_asset_price = self.query_historical_price(
'BTC', to_asset, timestamp)
price = asset_btc_price * btc_to_asset_price
else:
# attempt to get the daily price by timestamp
query_string = (
'https://min-api.cryptocompare.com/data/pricehistorical?'
'fsym={}&tsyms={}&ts={}'.format(from_asset, to_asset,
timestamp))
if to_asset == 'BTC':
query_string += '&tryConversion=false'
resp = urlopen(Request(query_string))
resp = rlk_jsonloads(resp.read())
print('DAILY PRICE OF ASSET: "{}"'.format(resp))
if from_asset not in resp:
error_message = 'Failed to query cryptocompare for: "{}"'.format(
query_string)
raise ValueError(error_message)
price = FVal(resp[from_asset][to_asset])
if price == 0:
raise NoPriceForGivenTimestamp(
from_asset, to_asset,
tsToDate(timestamp, formatstr='%d/%m/%Y, %H:%M:%S'))
return price
def get_history(self, start_ts, end_ts, end_at_least_ts=None):
"""Gets or creates trades and loans history from start_ts to end_ts or if
`end_at_least` is given and we have a cache history which satisfies it we
return the cache
"""
if end_at_least_ts is None:
end_at_least_ts = end_ts
historyfile_path = os.path.join(self.data_directory,
TRADES_HISTORYFILE)
if os.path.isfile(historyfile_path):
with open(historyfile_path, 'r') as infile:
try:
history_json_data = rlk_jsonloads(infile.read())
except:
pass
all_history_okay = data_up_todate(history_json_data, start_ts,
end_at_least_ts)
poloniex_history_okay = True
if self.poloniex is not None:
poloniex_history_okay = self.poloniex.check_trades_cache(
start_ts, end_at_least_ts) is not None
kraken_history_okay = True
if self.kraken is not None:
kraken_history_okay = self.kraken.check_trades_cache(
start_ts, end_at_least_ts) is not None
bittrex_history_okay = True
if self.bittrex is not None:
bittrex_history_okay = self.bittrex.check_trades_cache(
start_ts, end_at_least_ts) is not None
binance_history_okay = True
if self.binance is not None:
binance_history_okay = self.binance.check_trades_cache(
start_ts, end_at_least_ts) is not None
if not self.read_manual_margin_positions:
marginfile_path = os.path.join(self.data_directory,
MARGIN_HISTORYFILE)
margin_file_contents = get_jsonfile_contents_or_empty_dict(
marginfile_path)
margin_history_is_okay = data_up_todate(
margin_file_contents, start_ts, end_at_least_ts)
else:
margin_history_is_okay = True
margin_file_contents = do_read_manual_margin_positions(
self.data_directory)
loansfile_path = os.path.join(self.data_directory,
LOANS_HISTORYFILE)
loan_file_contents = get_jsonfile_contents_or_empty_dict(
loansfile_path)
loan_history_is_okay = data_up_todate(loan_file_contents,
start_ts,
end_at_least_ts)
assetmovementsfile_path = os.path.join(
self.data_directory, ASSETMOVEMENTS_HISTORYFILE)
asset_movements_contents = get_jsonfile_contents_or_empty_dict(
assetmovementsfile_path)
asset_movements_history_is_okay = data_up_todate(
asset_movements_contents, start_ts, end_at_least_ts)
eth_tx_log_path = os.path.join(self.data_directory,
ETHEREUM_TX_LOGFILE)
eth_tx_log_contents = get_jsonfile_contents_or_empty_dict(
eth_tx_log_path)
eth_tx_log_history_history_is_okay = data_up_todate(
eth_tx_log_contents, start_ts, end_at_least_ts)
if (all_history_okay and poloniex_history_okay
and kraken_history_okay and bittrex_history_okay
and binance_history_okay and margin_history_is_okay
and loan_history_is_okay
and asset_movements_history_is_okay
and eth_tx_log_history_history_is_okay):
history_trades = trades_from_dictlist(
history_json_data['data'], start_ts, end_ts)
if not self.read_manual_margin_positions:
margin_trades = trades_from_dictlist(
margin_file_contents['data'], start_ts, end_ts)
else:
margin_trades = margin_file_contents
eth_transactions = transactions_from_dictlist(
eth_tx_log_contents['data'], start_ts, end_ts)
asset_movements = asset_movements_from_dictlist(
asset_movements_contents['data'], start_ts, end_ts)
history_trades = include_external_trades(
self.data_directory, start_ts, end_ts, history_trades)
# make sure that this is the same as what is returned
# from create_history
return (history_trades, margin_trades,
loan_file_contents['data'], asset_movements,
eth_transactions)
return self.create_history(start_ts, end_ts, end_at_least_ts)
