def test_presto_fetch_error(monkeypatch):
monkeypatch.setattr(PrestoRequest.http.Response, "json",
get_json_get_error_0)
req = PrestoRequest(
host="coordinator",
port=8080,
user="******",
source="test",
catalog="test",
schema="test",
http_scheme="http",
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
with pytest.raises(prestodb.exceptions.PrestoUserError) as exception_info:
req.process(http_resp)
error = exception_info.value
assert error.error_code == 1
assert error.error_name == "SYNTAX_ERROR"
assert error.error_type == "USER_ERROR"
assert error.error_exception == "com.facebook.presto.sql.analyzer.SemanticException"
assert "stack" in error.failure_info
assert len(error.failure_info["stack"]) == 25
assert "suppressed" in error.failure_info
assert (
error.message ==
"line 1:15: Schema must be specified when session schema is not set")
assert error.error_location == (1, 15)
assert error.query_id == "20161116_205844_00002_xtnym"
def test_presto_fetch_error(monkeypatch):
monkeypatch.setattr(
PrestoRequest.http.Response,
'json',
get_json_get_error_0,
)
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
source='test',
catalog='test',
schema='test',
http_scheme='http',
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
with pytest.raises(prestodb.exceptions.PrestoUserError) as exception_info:
req.process(http_resp)
error = exception_info.value
assert error.error_code == 1
assert error.error_name == 'SYNTAX_ERROR'
assert error.error_type == 'USER_ERROR'
assert error.error_exception == 'com.facebook.presto.sql.analyzer.SemanticException'
assert 'stack' in error.failure_info
assert len(error.failure_info['stack']) == 25
assert 'suppressed' in error.failure_info
assert error.message == 'line 1:15: Schema must be specified when session schema is not set'
assert error.error_location == (1, 15)
assert error.query_id == '20161116_205844_00002_xtnym'
def test_gateway_redirect(monkeypatch):
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
gateway_response = FakeGatewayResponse(http_resp, redirect_count=3)
monkeypatch.setattr(PrestoRequest.http.Session, "post", gateway_response)
monkeypatch.setattr(socket, "gethostbyaddr", lambda *args:
("finalhost", ["finalhost"], "1.2.3.4"))
req = PrestoRequest(host="coordinator", port=8080, user="******")
result = req.post("http://host:80/path/")
assert gateway_response.count == 3
assert result.ok
def test_request_invalid_http_headers():
with pytest.raises(ValueError) as value_error:
PrestoRequest(
host='coordinator',
port=8080,
user='******',
http_headers={constants.HEADER_USER: '******'},
)
assert str(value_error.value).startswith('cannot override reserved HTTP header')
def test_request_invalid_http_headers():
with pytest.raises(ValueError) as value_error:
PrestoRequest(
host='coordinator',
port=8080,
user='******',
http_headers={constants.HEADER_PREFIX + 'Invalid': ''},
)
assert str(value_error.value).startswith('invalid HTTP header')
def test_presto_initial_request(monkeypatch):
monkeypatch.setattr(PrestoRequest.http.Response, "json", get_json_post_0)
req = PrestoRequest(
host="coordinator",
port=8080,
user="******",
source="test",
catalog="test",
schema="test",
http_scheme="http",
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
status = req.process(http_resp)
assert status.next_uri == RESP_DATA_POST_0["nextUri"]
assert status.id == RESP_DATA_POST_0["id"]
def test_presto_initial_request(monkeypatch):
monkeypatch.setattr(PrestoRequest.http.Response, 'json', get_json_post_0)
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
source='test',
catalog='test',
schema='test',
http_scheme='http',
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
status = req.process(http_resp)
assert status.next_uri == RESP_DATA_POST_0['nextUri']
assert status.id == RESP_DATA_POST_0['id']
def test_presto_connection_error(monkeypatch, error_code, error_type,
error_message):
monkeypatch.setattr(PrestoRequest.http.Response, "json", lambda x: {})
req = PrestoRequest(
host="coordinator",
port=8080,
user="******",
source="test",
catalog="test",
schema="test",
http_scheme="http",
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = error_code
with pytest.raises(error_type) as error:
req.process(http_resp)
assert error_message in str(error)
def test_gateway_redirect(monkeypatch):
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
gateway_response = FakeGatewayResponse(http_resp, redirect_count=3)
monkeypatch.setattr(PrestoRequest.http.Session, 'post', gateway_response)
monkeypatch.setattr(
socket,
'gethostbyaddr',
lambda *args: ('finalhost', ['finalhost'], '1.2.3.4'),
)
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
)
result = req.post('http://host:80/path/')
assert gateway_response.count == 3
assert result.ok
def test_presto_fetch_request_with_cookie(monkeypatch):
monkeypatch.setattr(PrestoRequest.http.Response, 'json', get_json_get_0)
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
source='test',
catalog='test',
schema='test',
http_scheme='http',
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
http_resp.cookies['key'] = 'value'
req.process(http_resp)
assert req._http_cookies['key'] == 'value'
def wait_for_presto_workers(host, port, timeout=30):
request = PrestoRequest(host=host, port=port, user="******")
sql = "SELECT state FROM system.runtime.nodes"
t0 = time.time()
while True:
query = PrestoQuery(request, sql)
rows = list(query.execute())
if any(row[0] == "active" for row in rows):
break
if time.time() - t0 > timeout:
raise TimeoutError
time.sleep(1)
def test_authentication_fail_retry(monkeypatch):
post_retry = RetryRecorder(error=KerberosExchangeError())
monkeypatch.setattr(PrestoRequest.http.Session, "post", post_retry)
get_retry = RetryRecorder(error=KerberosExchangeError())
monkeypatch.setattr(PrestoRequest.http.Session, "get", get_retry)
attempts = 3
kerberos_auth = KerberosAuthentication()
req = PrestoRequest(
host="coordinator",
port=8080,
user="******",
http_scheme=constants.HTTPS,
auth=kerberos_auth,
max_attempts=attempts,
)
with pytest.raises(KerberosExchangeError):
req.post("URL")
assert post_retry.retry_count == attempts
with pytest.raises(KerberosExchangeError):
req.get("URL")
assert post_retry.retry_count == attempts
def test_request_timeout():
timeout = 0.1
http_scheme = "http"
host = "coordinator"
port = 8080
url = http_scheme + "://" + host + ":" + str(
port) + constants.URL_STATEMENT_PATH
def long_call(request, uri, headers):
time.sleep(timeout * 2)
return (200, headers, "delayed success")
httpretty.enable()
for method in [httpretty.POST, httpretty.GET]:
httpretty.register_uri(method, url, body=long_call)
# timeout without retry
for request_timeout in [timeout, (timeout, timeout)]:
req = PrestoRequest(
host=host,
port=port,
user="******",
http_scheme=http_scheme,
max_attempts=1,
request_timeout=request_timeout,
)
with pytest.raises(requests.exceptions.Timeout):
req.get(url)
with pytest.raises(requests.exceptions.Timeout):
req.post("select 1")
httpretty.disable()
httpretty.reset()
def test_presto_fetch_error(monkeypatch):
monkeypatch.setattr(
PrestoRequest.http.Response,
'json',
get_json_get_error_0,
)
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
source='test',
catalog='test',
schema='test',
http_scheme='http',
session_properties={},
)
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 200
with pytest.raises(prestodb.exceptions.PrestoUserError):
req.process(http_resp)
def test_request_headers(monkeypatch):
post_recorder = ArgumentsRecorder()
monkeypatch.setattr(PrestoRequest.http.Session, "post", post_recorder)
get_recorder = ArgumentsRecorder()
monkeypatch.setattr(PrestoRequest.http.Session, "get", get_recorder)
catalog = "test_catalog"
schema = "test_schema"
user = "******"
source = "test_source"
accept_encoding_header = "accept-encoding"
accept_encoding_value = "identity,deflate,gzip"
client_info_header = constants.HEADER_PREFIX + "Client-Info"
client_info_value = "some_client_info"
req = PrestoRequest(
host="coordinator",
port=8080,
user=user,
source=source,
catalog=catalog,
schema=schema,
http_scheme="http",
session_properties={
"hash_partition_count": 500,
"needs_url_encoding": 'foo,bar'
},
http_headers={
accept_encoding_header: accept_encoding_value,
client_info_header: client_info_value,
},
redirect_handler=None,
)
def assert_headers(headers):
assert headers[constants.HEADER_CATALOG] == catalog
assert headers[constants.HEADER_SCHEMA] == schema
assert headers[constants.HEADER_SOURCE] == source
assert headers[constants.HEADER_USER] == user
assert headers[constants.HEADER_SESSION] in (
'hash_partition_count=500,needs_url_encoding=foo%2Cbar',
'needs_url_encoding=foo%2Cbar,hash_partition_count=500')
assert headers[accept_encoding_header] == accept_encoding_value
assert headers[client_info_header] == client_info_value
assert len(headers.keys()) == 8
req.post("URL")
assert_headers(post_recorder.kwargs["headers"])
req.get("URL")
assert_headers(get_recorder.kwargs["headers"])
def test_request_headers(monkeypatch):
post_recorder = ArgumentsRecorder()
monkeypatch.setattr(PrestoRequest.http.Session, 'post', post_recorder)
get_recorder = ArgumentsRecorder()
monkeypatch.setattr(PrestoRequest.http.Session, 'get', get_recorder)
catalog = 'test_catalog'
schema = 'test_schema'
user = '******'
source = 'test_source'
accept_encoding_header = 'accept-encoding'
accept_encoding_value = 'identity,deflate,gzip'
client_info_header = constants.HEADER_PREFIX + 'Client-Info'
client_info_value = 'some_client_info'
req = PrestoRequest(
host='coordinator',
port=8080,
user=user,
source=source,
catalog=catalog,
schema=schema,
http_scheme='http',
session_properties={},
http_headers={
accept_encoding_header: accept_encoding_value,
client_info_header: client_info_value,
},
redirect_handler=None,
)
def assert_headers(headers):
assert headers[constants.HEADER_CATALOG] == catalog
assert headers[constants.HEADER_SCHEMA] == schema
assert headers[constants.HEADER_SOURCE] == source
assert headers[constants.HEADER_USER] == user
assert headers[constants.HEADER_SESSION] == ''
assert headers[accept_encoding_header] == accept_encoding_value
assert headers[client_info_header] == client_info_value
assert len(headers.keys()) == 8
req.post('URL')
assert_headers(post_recorder.kwargs['headers'])
req.get('URL')
assert_headers(get_recorder.kwargs['headers'])
def test_503_error_retry(monkeypatch):
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 503
post_retry = RetryRecorder(result=http_resp)
monkeypatch.setattr(PrestoRequest.http.Session, "post", post_retry)
get_retry = RetryRecorder(result=http_resp)
monkeypatch.setattr(PrestoRequest.http.Session, "get", get_retry)
attempts = 3
req = PrestoRequest(host="coordinator",
port=8080,
user="******",
max_attempts=attempts)
req.post("URL")
assert post_retry.retry_count == attempts
req.get("URL")
assert post_retry.retry_count == attempts
def test_authentication_fail_retry(monkeypatch):
post_retry = RetryRecorder(error=KerberosExchangeError())
monkeypatch.setattr(PrestoRequest.http.Session, 'post', post_retry)
get_retry = RetryRecorder(error=KerberosExchangeError())
monkeypatch.setattr(PrestoRequest.http.Session, 'get', get_retry)
attempts = 3
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
max_attempts=attempts,
)
with pytest.raises(KerberosExchangeError):
req.post('URL')
assert post_retry.retry_count == attempts
with pytest.raises(KerberosExchangeError):
req.get('URL')
assert post_retry.retry_count == attempts
def test_503_error_retry(monkeypatch):
http_resp = PrestoRequest.http.Response()
http_resp.status_code = 503
post_retry = RetryRecorder(result=http_resp)
monkeypatch.setattr(PrestoRequest.http.Session, 'post', post_retry)
get_retry = RetryRecorder(result=http_resp)
monkeypatch.setattr(PrestoRequest.http.Session, 'get', get_retry)
attempts = 3
req = PrestoRequest(
host='coordinator',
port=8080,
user='******',
max_attempts=attempts,
)
req.post('URL')
assert post_retry.retry_count == attempts
req.get('URL')
assert post_retry.retry_count == attempts