def __init__(self, timeout, _include_retry_params=False): self.timeout = timeout self.cnt = 0 self.sleeping_time = 1 self.start_time = get_time_millis() self._include_retry_params = _include_retry_params # backoff between 1 and 16 seconds self._backoff = DecorrelateJitterBackoff(1, 16)
def _request_exec(self, session, method, full_url, headers, data, proxies, token, catch_okta_unauthorized_error=False, is_raw_text=False, is_raw_binary=False, is_raw_binary_iterator=True, use_ijson=False, socket_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT, return_timing_metrics=False): if socket_timeout > DEFAULT_SOCKET_CONNECT_TIMEOUT: # socket timeout should not be more than the default. # A shorter timeout may be specified for login time, but # for query, it should be at least 45 seconds. socket_timeout = DEFAULT_SOCKET_CONNECT_TIMEOUT logger.debug('socket timeout: %s', socket_timeout) try: if not catch_okta_unauthorized_error and data and len(data) > 0: gzdata = BytesIO() gzip.GzipFile(fileobj=gzdata, mode=u'wb').write(data.encode(u'utf-8')) gzdata.seek(0, 0) headers['Content-Encoding'] = 'gzip' input_data = gzdata else: input_data = data timing_metrics = {} start_time = get_time_millis() # socket timeout is constant. You should be able to receive # the response within the time. If not, ConnectReadTimeout or # ReadTimeout is raised. raw_ret = session.request( method=method, url=full_url, proxies=proxies, headers=headers, data=input_data, timeout=socket_timeout, verify=True, stream=is_raw_binary, auth=SnowflakeAuth(token), ) timing_metrics[ResultIterWithTimings. DOWNLOAD] = get_time_millis() - start_time try: if raw_ret.status_code == OK: logger.debug(u'SUCCESS') if is_raw_text: ret = raw_ret.text elif is_raw_binary: start_time = get_time_millis() raw_data = decompress_raw_data( raw_ret.raw, add_bracket=True).decode('utf-8', 'replace') if not is_raw_binary_iterator: ret = json.loads(raw_data) elif not use_ijson: ret = iter(json.loads(raw_data)) else: ret = split_rows_from_stream(StringIO(raw_data)) timing_metrics[ResultIterWithTimings. PARSE] = get_time_millis() - start_time # if timings requested, wrap the iterator so the timing # info is accessible if return_timing_metrics: ret = ResultIterWithTimings(ret, timing_metrics) else: ret = raw_ret.json() return ret if is_retryable_http_code(raw_ret.status_code): ex = STATUS_TO_EXCEPTION.get(raw_ret.status_code, OtherHTTPRetryableError) exi = ex(code=raw_ret.status_code) logger.debug('%s. Retrying...', exi) # retryable server exceptions raise RetryRequest(exi) elif raw_ret.status_code == UNAUTHORIZED and \ catch_okta_unauthorized_error: # OKTA Unauthorized errors Error.errorhandler_wrapper( self._connection, None, DatabaseError, { u'msg': (u'Failed to get authentication by OKTA: ' u'{status}: {reason}').format( status=raw_ret.status_code, reason=raw_ret.reason), u'errno': ER_FAILED_TO_CONNECT_TO_DB, u'sqlstate': SQLSTATE_CONNECTION_REJECTED }) return None # required for tests else: Error.errorhandler_wrapper( self._connection, None, InterfaceError, { u'msg': (u"{status} {reason}: " u"{method} {url}").format( status=raw_ret.status_code, reason=raw_ret.reason, method=method, url=full_url), u'errno': ER_FAILED_TO_REQUEST, u'sqlstate': SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED }) return None # required for tests finally: raw_ret.close() # ensure response is closed except ( ConnectTimeout, ReadTimeout, BadStatusLine, ConnectionError, SSLError, ProtocolError, # from urllib3 ReadTimeoutError, # from urllib3 OpenSSL.SSL.SysCallError, KeyError, # SNOW-39175: asn1crypto.keys.PublicKeyInfo ValueError, RuntimeError, AttributeError, # json decoding error ) as err: logger.debug( "Hit retryable client error. Retrying... Ignore the following " "error stack: %s", err, exc_info=True) raise RetryRequest(err) except Exception as err: if PY2 and isinstance(err, PyAsn1Error): logger.debug( "Hit retryable client error. Retrying... " "Ignore the following error stack: %s", err, exc_info=True) raise RetryRequest(err) raise err
def _request_exec(self, session, method, full_url, headers, data, token, catch_okta_unauthorized_error=False, is_raw_text=False, is_raw_binary=False, binary_data_handler=None, socket_timeout=DEFAULT_SOCKET_CONNECT_TIMEOUT): if socket_timeout > DEFAULT_SOCKET_CONNECT_TIMEOUT: # socket timeout should not be more than the default. # A shorter timeout may be specified for login time, but # for query, it should be at least 45 seconds. socket_timeout = DEFAULT_SOCKET_CONNECT_TIMEOUT logger.debug('socket timeout: %s', socket_timeout) try: if not catch_okta_unauthorized_error and data and len(data) > 0: gzdata = BytesIO() gzip.GzipFile(fileobj=gzdata, mode='wb').write(data.encode('utf-8')) gzdata.seek(0, 0) headers['Content-Encoding'] = 'gzip' input_data = gzdata else: input_data = data download_start_time = get_time_millis() # socket timeout is constant. You should be able to receive # the response within the time. If not, ConnectReadTimeout or # ReadTimeout is raised. raw_ret = session.request( method=method, url=full_url, headers=headers, data=input_data, timeout=socket_timeout, verify=True, stream=is_raw_binary, auth=SnowflakeAuth(token), ) download_end_time = get_time_millis() try: if raw_ret.status_code == OK: logger.debug('SUCCESS') if is_raw_text: ret = raw_ret.text elif is_raw_binary: ret = binary_data_handler.to_iterator( raw_ret.raw, download_end_time - download_start_time) else: ret = raw_ret.json() return ret if is_retryable_http_code(raw_ret.status_code): ex = STATUS_TO_EXCEPTION.get(raw_ret.status_code, OtherHTTPRetryableError) exi = ex(code=raw_ret.status_code) logger.debug('%s. Retrying...', exi) # retryable server exceptions raise RetryRequest(exi) elif raw_ret.status_code == UNAUTHORIZED and \ catch_okta_unauthorized_error: # OKTA Unauthorized errors Error.errorhandler_wrapper( self._connection, None, DatabaseError, { 'msg': ('Failed to get authentication by OKTA: ' '{status}: {reason}').format( status=raw_ret.status_code, reason=raw_ret.reason), 'errno': ER_FAILED_TO_CONNECT_TO_DB, 'sqlstate': SQLSTATE_CONNECTION_REJECTED }) return None # required for tests else: TelemetryService.get_instance().log_http_request_error( "HttpError%s" % str(raw_ret.status_code), full_url, method, SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED, ER_FAILED_TO_REQUEST, response=raw_ret) Error.errorhandler_wrapper( self._connection, None, InterfaceError, { 'msg': ("{status} {reason}: " "{method} {url}").format( status=raw_ret.status_code, reason=raw_ret.reason, method=method, url=full_url), 'errno': ER_FAILED_TO_REQUEST, 'sqlstate': SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED }) return None # required for tests finally: raw_ret.close() # ensure response is closed except SSLError as se: logger.debug("Hit non-retryable SSL error, %s", str(se)) TelemetryService.get_instance().log_http_request_error( "CertificateException%s" % str(se), full_url, method, SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED, ER_FAILED_TO_REQUEST, exception=se, stack_trace=traceback.format_exc()) except ( ConnectTimeout, ReadTimeout, BadStatusLine, ConnectionError, IncompleteRead, ProtocolError, # from urllib3 ReadTimeoutError, # from urllib3 OpenSSL.SSL.SysCallError, KeyError, # SNOW-39175: asn1crypto.keys.PublicKeyInfo ValueError, RuntimeError, AttributeError, # json decoding error ) as err: logger.debug( "Hit retryable client error. Retrying... Ignore the following " "error stack: %s", err, exc_info=True) raise RetryRequest(err) except Exception as err: _, _, stack_trace = sys.exc_info() TelemetryService.get_instance().log_http_request_error( "HttpException%s" % str(err), full_url, method, SQLSTATE_CONNECTION_WAS_NOT_ESTABLISHED, ER_FAILED_TO_REQUEST, exception=err, stack_trace=traceback.format_exc()) raise err