def _curl_bitmex(self,
path,
query=None,
postdict=None,
timeout=None,
verb=None,
rethrow_errors=False,
max_retries=None):
"""Send a request to BitMEX Servers."""
# Handle URL
url = self.host + path
if timeout is None:
timeout = self.timeout
# Default to POST if data is attached, GET otherwise
if not verb:
verb = 'POST' if postdict else 'GET'
# By default don't retry POST or PUT. Retrying GET/DELETE is okay because they are idempotent.
# In the future we could allow retrying PUT, so long as 'leavesQty' is not used (not idempotent),
# or you could change the clOrdID (set {"clOrdID": "new", "origClOrdID": "old"}) so that an amend
# can't erroneously be applied twice.
if max_retries is None:
max_retries = 0 if verb in ['POST', 'PUT'] else 3
def exit_or_throw(e):
self.retries = 0
if rethrow_errors:
raise e
else:
exit(1)
# def retry():
# self.retries += 1
# if self.retries > max_retries:
# raise Exception("Max retries on %s (%s) hit, raising." % (path, json.dumps(postdict or '')))
# return self._curl_bitmex(path, query, postdict, timeout, verb, rethrow_errors, max_retries)
while self.retries <= max_retries:
if self.retries > 0:
self.logger.info("retry request")
# Auth: API Key/Secret
auth = APIKeyAuthWithExpires(self.api_key, self.api_secret)
# Make the request
response = None
try:
# self.logger.info("sending req to %s: %s" % (url, json.dumps(postdict or query or '')))
print("sending req to %s: %s" %
(url, json.dumps(postdict or query or '')))
req = requests.Request(verb,
url,
json=postdict,
auth=auth,
params=query)
prepped = self.session.prepare_request(req)
response = self.session.send(prepped, timeout=timeout)
# Make non-200s throw
response.raise_for_status()
except requests.exceptions.HTTPError as e:
if response is None:
self.logger.error("Response is null, %s" % e)
self.retries += 1
continue
# 401 - Auth error. This is fatal.
if response.status_code == 401:
self.logger.error(
"API Key or Secret incorrect, please check and restart."
)
self.logger.error("Error: " + response.text)
if postdict:
self.logger.error(postdict)
# Always exit, even if rethrow_errors, because this is fatal
# exit(1)
exit_or_throw(e)
# 404, can be thrown if order canceled or does not exist.
elif response.status_code == 404:
if verb == 'DELETE':
self.logger.error("Order not found: %s" %
postdict['orderID'])
return
self.logger.error(
"Unable to contact the BitMEX API (404). " +
"Request: %s \n %s" % (url, json.dumps(postdict)))
exit_or_throw(e)
# 429, ratelimit; cancel orders & wait until X-RateLimit-Reset
elif response.status_code == 429:
self.logger.error(
"Ratelimited on current request. Sleeping, then trying again. Try fewer "
+
"order pairs or contact [email protected] to raise your limits. "
+ "Request: %s \n %s" % (url, json.dumps(postdict)))
# Figure out how long we need to wait.
ratelimit_reset = response.headers['X-RateLimit-Reset']
to_sleep = int(ratelimit_reset) - int(time.time())
reset_str = datetime.datetime.fromtimestamp(
int(ratelimit_reset)).strftime('%X')
# We're ratelimited, and we may be waiting for a long time. Cancel orders.
self.logger.warning(
"Canceling all known orders in the meantime.")
self.cancel([o['orderID'] for o in self.open_orders()])
self.logger.error(
"Your ratelimit will reset at %s. Sleeping for %d seconds."
% (reset_str, to_sleep))
time.sleep(to_sleep)
# Retry the request.
self.retries += 1
continue
# 503 - BitMEX temporary downtime, likely due to a deploy. Try again
elif response.status_code == 503:
self.logger.warning(
"Unable to contact the BitMEX API (503), retrying. " +
"Request: %s \n %s" % (url, json.dumps(postdict)))
# sleep 5 sec
time.sleep(3)
time.sleep(2)
# set max_retries to 12
max_retries = 12.
# Retry the request.
self.retries += 1
continue
elif response.status_code == 400:
error = response.json()['error']
message = error['message'].lower() if error else ''
# Duplicate clOrdID: that's fine, probably a deploy, go get the order(s) and return it
if 'duplicate clordid' in message:
orders = postdict[
'orders'] if 'orders' in postdict else postdict
IDs = json.dumps({
'clOrdID': [order['clOrdID'] for order in orders]
})
orderResults = self._curl_bitmex('/order',
query={'filter': IDs},
verb='GET')
for i, order in enumerate(orderResults):
if (order['orderQty'] != abs(postdict['orderQty'])
or order['side'] !=
('Buy' if postdict['orderQty'] > 0 else 'Sell')
or order['price'] != postdict['price']
or order['symbol'] != postdict['symbol']):
self.retries = 0
raise Exception(
'Attempted to recover from duplicate clOrdID, but order returned from API '
+
'did not match POST.\nPOST data: %s\nReturned order: %s'
%
(json.dumps(orders[i]), json.dumps(order)))
# All good
return orderResults
elif 'insufficient available balance' in message:
self.logger.error(
'Account out of funds. The message: %s' %
error['message'])
exit_or_throw(Exception('Insufficient Funds'))
# If we haven't returned or re-raised yet, we get here.
self.logger.error("Unhandled Error: %s: %s" %
(e, response.text))
self.logger.error("Endpoint was: %s %s: %s" %
(verb, path, json.dumps(postdict)))
# exit_or_throw(e)
self.retries += 1
continue
except requests.exceptions.Timeout as e:
# Timeout, re-run this request
self.logger.warning(
"Timed out on request: %s (%s), retrying..." %
(path, json.dumps(postdict or '')))
# return retry()
self.retries += 1
continue
except requests.exceptions.ConnectionError as e:
self.logger.warning(
"Unable to contact the BitMEX API (%s). Please check the URL. Retrying. "
+ "Request: %s %s \n %s" % (e, url, json.dumps(postdict)))
time.sleep(1)
# return retry()
self.retries += 1
continue
# Reset retry counter on success
self.retries = 0
return response.json()
self.retries = 0
raise Exception("Max retries on %s (%s) hit, raising." %
(path, json.dumps(postdict or '')))
def _curl_bitmex(self, api, query=None, postdict=None, timeout=10, verb=None, rethrow_errors=True):
"""Send a request to BitMEX Servers."""
# Handle URL
url = self.base_url + api
# Default to POST if data is attached, GET otherwise
if not verb:
verb = 'POST' if postdict else 'GET'
# Auth: Use Access Token by default, API Key/Secret if provided
auth = AccessTokenAuth(self.token)
if self.apiKey:
auth = APIKeyAuthWithExpires(self.apiKey, self.apiSecret)
def maybe_exit(e):
if rethrow_errors:
raise e
else:
exit(1)
# Make the request
try:
self.pending_requests += 1
now = getUTCtime()
# Rate limiting in bursts
diff = self.reset - now
if not self.remaining:
self.remaining = 0.001
diff = 180
if (diff / self.remaining) > 1:
# self.logger.debug("remaining: %s / till reset: %s" % (self.remaining, diff))
delay = round(diff - self.remaining) + self.pending_requests + 1
# self.logger.debug("sleep delay %d pending %d" % (delay, self.pending_requests))
sleep(delay)
req = requests.Request(verb, url, json=postdict, auth=auth, params=query)
self.pending_requests -= 1
prepped = self.session.prepare_request(req)
response = self.session.send(prepped, timeout=timeout)
self.remaining = float(
response.headers[
'X-RateLimit-Remaining']) - 0.01 if 'X-RateLimit-Remaining' in response.headers else 0.001
self.reset = int(
response.headers['X-RateLimit-Reset']) if 'X-RateLimit-Reset' in response.headers else now + 180
# Make non-200s throw
response.raise_for_status()
except requests.exceptions.HTTPError as e:
# 401 - Auth error. This is fatal with API keys.
if (response.json()['error'] and response.json()['error'][
'message'] == '2FA Token is required and did not match.'):
return response.json()
elif response.status_code == 401:
self.logger.error("Login information or API Key incorrect, please check and restart.")
self.logger.error("Error: " + response.text)
if postdict:
self.logger.error(postdict)
# Always exit, even if rethrow_errors, because this is fatal
raise e
return response.text
# return self._curl_bitmex(api, query, postdict, timeout, verb)
# Unknown Error
else:
self.logger.error("Unhandled Error: %s: %s" % (e, response.text))
self.logger.error("Endpoint was: %s %s: %s" % (verb, api, json.dumps(postdict)))
maybe_exit(e)
return response.text
except Exception as e:
return str(e)
return response.json()
def _curl_bitmex(self,
api,
query=None,
postdict=None,
timeout=3,
verb=None):
"""Send a request to BitMEX Servers."""
# Handle URL
url = self.base_url + api
# Default to POST if data is attached, GET otherwise
if not verb:
verb = 'POST' if postdict else 'GET'
# Auth: Use Access Token by default, API Key/Secret if provided
auth = AccessTokenAuth(self.token)
if self.apiKey:
auth = APIKeyAuthWithExpires(self.apiKey, self.apiSecret)
# Make the request
try:
req = requests.Request(verb,
url,
data=postdict,
auth=auth,
params=query)
prepped = self.session.prepare_request(req)
response = self.session.send(prepped, timeout=timeout)
# Make non-200s throw
response.raise_for_status()
except requests.exceptions.HTTPError, e:
# 401 - Auth error. Re-auth and re-run this request.
if response.status_code == 401:
if self.token is None:
self.logger.error(
"Login information or API Key incorrect, please check and restart."
)
self.logger.error("Error: " + response.text)
if postdict:
self.logger.error(postdict)
exit(1)
self.logger.warning("Token expired, reauthenticating...")
sleep(1)
self.authenticate()
return self._curl_bitmex(api, query, postdict, timeout, verb)
# 404, can be thrown if order canceled does not exist.
elif response.status_code == 404:
if verb == 'DELETE':
self.logger.error("Order not found: %s" %
postdict['orderID'])
return
self.logger.error("Unable to contact the BitMEX API (404). " +
"Request: %s \n %s" %
(url, json.dumps(postdict)))
exit(1)
# 429, ratelimit
elif response.status_code == 429:
self.logger.error(
"Ratelimited on current request. Sleeping, then trying again. Try fewer "
+
"order pairs or contact [email protected] to raise your limits. "
+ "Request: %s \n %s" % (url, json.dumps(postdict)))
sleep(1)
return self._curl_bitmex(api, query, postdict, timeout, verb)
# 503 - BitMEX temporary downtime, likely due to a deploy. Try again
elif response.status_code == 503:
self.logger.warning(
"Unable to contact the BitMEX API (503), retrying. " +
"Request: %s \n %s" % (url, json.dumps(postdict)))
sleep(1)
return self._curl_bitmex(api, query, postdict, timeout, verb)
# Unknown Error
else:
self.logger.error(
"Unhandled Error: %s: %s %s" %
(e, e.message, json.dumps(response.json(), indent=4)))
self.logger.error("Endpoint was: %s %s" % (verb, api))
exit(1)
def get_bucketed_trades(
apiKey,
apiSecret,
url,
Q=None,
fout=None,
startTime=None,
endTime=None,
binSize="1d",
pause=0.5,
reverse="false",
):
"""
Returns the historical data for XBTUSD (default) from `startTime` to `endTime` bucketed by `binSize`
return columns are: timestamp, symbol, open, high, low, close, trades, volume, vwap, lastSize,
turnover, homeNotional, foreignNotional,
- binSize (str) is one of 1m, 5m, 1h, 1d
- Time are in isoformat eg. 2016-12-27T11:00Z
Params:
apiKey, apiSecret, url, obvious
Q=None, The Query requested passed as a dictionnary with keys binSize, partial, symbol, count and reverse.
fout=None, the name of the file to write to
pause=0.5, to throttle the request
reverse="false",
"""
logger.debug(
f"Got {Q}, fout={fout}, startTime={startTime}, endTime={endTime},"
f" binSize={binSize}, pause={pause}")
# Init session and defaults settings
auth = APIKeyAuthWithExpires(apiKey, apiSecret)
sess = init_session()
fout = "./btxData.csv" if fout is None else fout
Q = ({
"binSize": binSize,
"partial": "false",
"symbol": "XBTUSD",
"count": 180,
"reverse": reverse,
} if Q is None else Q)
if startTime is not None:
Q["startTime"] = pd.Timestamp(startTime).round(TC[binSize])
else:
startTime = Q["startTime"]
# Ready to open the file make requests and write results
with open(fout, "w") as fd:
Q, firstReqDate, lastReqDate = request_write_nlog(Q,
sess,
auth,
url,
fd,
header=True,
pause=0)
logging.warning(f"Req 0: Q={Q}, {firstReqDate.strftime(STRF)}"
f" --> {lastReqDate.strftime(STRF)}")
i = 1
while not reached(lastReqDate, endTime):
Q, firstReqDate, lastReqDate = request_write_nlog(
Q,
sess,
auth,
url,
fd,
step=i,
startTime=lastReqDate,
pause=pause)
print(
f"Req {i}: {firstReqDate.strftime(STRF)}"
f" --> {lastReqDate.strftime(STRF)}",
end="\r",
)
i += 1
# last log before exit
Q["startTime"], Q["endTime"] = startTime, endTime
logging.warning(f"Finished in {i} requests for query={Q}")
return sess