def __init__(self, timeout, _include_retry_params=False):
self.timeout = timeout
self.cnt = 0
self.sleeping_time = 1
self.start_time = get_time_millis()
self._include_retry_params = _include_retry_params
# backoff between 1 and 16 seconds
self._backoff = DecorrelateJitterBackoff(1, 16)
def _request_exec(self,
session,
method,
full_url,
headers,
data,
proxies,
token,
catch_okta_unauthorized_error=False,
is_raw_text=False,
is_raw_binary=False,
is_raw_binary_iterator=True,
use_ijson=False,
socket_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT,
return_timing_metrics=False):
if socket_timeout > DEFAULT_SOCKET_CONNECT_TIMEOUT:
# socket timeout should not be more than the default.
# A shorter timeout may be specified for login time, but
# for query, it should be at least 45 seconds.
socket_timeout = DEFAULT_SOCKET_CONNECT_TIMEOUT
logger.debug('socket timeout: %s', socket_timeout)
try:
if not catch_okta_unauthorized_error and data and len(data) > 0:
gzdata = BytesIO()
gzip.GzipFile(fileobj=gzdata,
mode=u'wb').write(data.encode(u'utf-8'))
gzdata.seek(0, 0)
headers['Content-Encoding'] = 'gzip'
input_data = gzdata
else:
input_data = data
timing_metrics = {}
start_time = get_time_millis()
# socket timeout is constant. You should be able to receive
# the response within the time. If not, ConnectReadTimeout or
# ReadTimeout is raised.
raw_ret = session.request(
method=method,
url=full_url,
proxies=proxies,
headers=headers,
data=input_data,
timeout=socket_timeout,
verify=True,
stream=is_raw_binary,
auth=SnowflakeAuth(token),
)
timing_metrics[ResultIterWithTimings.
DOWNLOAD] = get_time_millis() - start_time
try:
if raw_ret.status_code == OK:
logger.debug(u'SUCCESS')
if is_raw_text:
ret = raw_ret.text
elif is_raw_binary:
start_time = get_time_millis()
raw_data = decompress_raw_data(
raw_ret.raw,
add_bracket=True).decode('utf-8', 'replace')
if not is_raw_binary_iterator:
ret = json.loads(raw_data)
elif not use_ijson:
ret = iter(json.loads(raw_data))
else:
ret = split_rows_from_stream(StringIO(raw_data))
timing_metrics[ResultIterWithTimings.
PARSE] = get_time_millis() - start_time
# if timings requested, wrap the iterator so the timing
# info is accessible
if return_timing_metrics:
ret = ResultIterWithTimings(ret, timing_metrics)
else:
ret = raw_ret.json()
return ret
if is_retryable_http_code(raw_ret.status_code):
ex = STATUS_TO_EXCEPTION.get(raw_ret.status_code,
OtherHTTPRetryableError)
exi = ex(code=raw_ret.status_code)
logger.debug('%s. Retrying...', exi)
# retryable server exceptions
raise RetryRequest(exi)
elif raw_ret.status_code == UNAUTHORIZED and \
catch_okta_unauthorized_error:
# OKTA Unauthorized errors
Error.errorhandler_wrapper(
self._connection, None, DatabaseError, {
u'msg': (u'Failed to get authentication by OKTA: '
u'{status}: {reason}').format(
status=raw_ret.status_code,
reason=raw_ret.reason),
u'errno':
ER_FAILED_TO_CONNECT_TO_DB,
u'sqlstate':
SQLSTATE_CONNECTION_REJECTED
})
return None # required for tests
else:
Error.errorhandler_wrapper(
self._connection, None, InterfaceError, {
u'msg': (u"{status} {reason}: "
u"{method} {url}").format(
status=raw_ret.status_code,
reason=raw_ret.reason,
method=method,
url=full_url),
u'errno':
ER_FAILED_TO_REQUEST,
u'sqlstate':
SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED
})
return None # required for tests
finally:
raw_ret.close() # ensure response is closed
except (
ConnectTimeout,
ReadTimeout,
BadStatusLine,
ConnectionError,
SSLError,
ProtocolError, # from urllib3
ReadTimeoutError, # from urllib3
OpenSSL.SSL.SysCallError,
KeyError, # SNOW-39175: asn1crypto.keys.PublicKeyInfo
ValueError,
RuntimeError,
AttributeError, # json decoding error
) as err:
logger.debug(
"Hit retryable client error. Retrying... Ignore the following "
"error stack: %s",
err,
exc_info=True)
raise RetryRequest(err)
except Exception as err:
if PY2 and isinstance(err, PyAsn1Error):
logger.debug(
"Hit retryable client error. Retrying... "
"Ignore the following error stack: %s",
err,
exc_info=True)
raise RetryRequest(err)
raise err
def _request_exec(self,
session,
method,
full_url,
headers,
data,
token,
catch_okta_unauthorized_error=False,
is_raw_text=False,
is_raw_binary=False,
binary_data_handler=None,
socket_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT):
if socket_timeout > DEFAULT_SOCKET_CONNECT_TIMEOUT:
# socket timeout should not be more than the default.
# A shorter timeout may be specified for login time, but
# for query, it should be at least 45 seconds.
socket_timeout = DEFAULT_SOCKET_CONNECT_TIMEOUT
logger.debug('socket timeout: %s', socket_timeout)
try:
if not catch_okta_unauthorized_error and data and len(data) > 0:
gzdata = BytesIO()
gzip.GzipFile(fileobj=gzdata,
mode='wb').write(data.encode('utf-8'))
gzdata.seek(0, 0)
headers['Content-Encoding'] = 'gzip'
input_data = gzdata
else:
input_data = data
download_start_time = get_time_millis()
# socket timeout is constant. You should be able to receive
# the response within the time. If not, ConnectReadTimeout or
# ReadTimeout is raised.
raw_ret = session.request(
method=method,
url=full_url,
headers=headers,
data=input_data,
timeout=socket_timeout,
verify=True,
stream=is_raw_binary,
auth=SnowflakeAuth(token),
)
download_end_time = get_time_millis()
try:
if raw_ret.status_code == OK:
logger.debug('SUCCESS')
if is_raw_text:
ret = raw_ret.text
elif is_raw_binary:
ret = binary_data_handler.to_iterator(
raw_ret.raw,
download_end_time - download_start_time)
else:
ret = raw_ret.json()
return ret
if is_retryable_http_code(raw_ret.status_code):
ex = STATUS_TO_EXCEPTION.get(raw_ret.status_code,
OtherHTTPRetryableError)
exi = ex(code=raw_ret.status_code)
logger.debug('%s. Retrying...', exi)
# retryable server exceptions
raise RetryRequest(exi)
elif raw_ret.status_code == UNAUTHORIZED and \
catch_okta_unauthorized_error:
# OKTA Unauthorized errors
Error.errorhandler_wrapper(
self._connection, None, DatabaseError, {
'msg': ('Failed to get authentication by OKTA: '
'{status}: {reason}').format(
status=raw_ret.status_code,
reason=raw_ret.reason),
'errno':
ER_FAILED_TO_CONNECT_TO_DB,
'sqlstate':
SQLSTATE_CONNECTION_REJECTED
})
return None # required for tests
else:
TelemetryService.get_instance().log_http_request_error(
"HttpError%s" % str(raw_ret.status_code),
full_url,
method,
SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED,
ER_FAILED_TO_REQUEST,
response=raw_ret)
Error.errorhandler_wrapper(
self._connection, None, InterfaceError, {
'msg': ("{status} {reason}: "
"{method} {url}").format(
status=raw_ret.status_code,
reason=raw_ret.reason,
method=method,
url=full_url),
'errno':
ER_FAILED_TO_REQUEST,
'sqlstate':
SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED
})
return None # required for tests
finally:
raw_ret.close() # ensure response is closed
except SSLError as se:
logger.debug("Hit non-retryable SSL error, %s", str(se))
TelemetryService.get_instance().log_http_request_error(
"CertificateException%s" % str(se),
full_url,
method,
SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED,
ER_FAILED_TO_REQUEST,
exception=se,
stack_trace=traceback.format_exc())
except (
ConnectTimeout,
ReadTimeout,
BadStatusLine,
ConnectionError,
IncompleteRead,
ProtocolError, # from urllib3
ReadTimeoutError, # from urllib3
OpenSSL.SSL.SysCallError,
KeyError, # SNOW-39175: asn1crypto.keys.PublicKeyInfo
ValueError,
RuntimeError,
AttributeError, # json decoding error
) as err:
logger.debug(
"Hit retryable client error. Retrying... Ignore the following "
"error stack: %s",
err,
exc_info=True)
raise RetryRequest(err)
except Exception as err:
_, _, stack_trace = sys.exc_info()
TelemetryService.get_instance().log_http_request_error(
"HttpException%s" % str(err),
full_url,
method,
SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED,
ER_FAILED_TO_REQUEST,
exception=err,
stack_trace=traceback.format_exc())
raise err
