def test_get_profile(self):
statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# If ExecuteStatement() has completed but the results haven't been fetched yet, the
# query must have reached either COMPILED or RUNNING or FINISHED.
assert "Query State: COMPILED" in get_profile_resp.profile or \
"Query State: RUNNING" in get_profile_resp.profile or \
"Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# After fetching the results, we must be in state FINISHED.
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
# Test that session secret is validated. Note that operation secret does not need to
# be validated in addition if the session secret is valid and the operation belongs
# to the session, because the user has full access to the session.
TestHS2.check_invalid_session(
self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle,
create_session_handle_without_secret(
self.session_handle))))
# Attempt to access query with different user should fail.
evil_user = getuser() + "_evil_twin"
session_handle2 = self._open_extra_session(evil_user)
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(
self.hs2_client.CloseOperation(close_operation_req))
# Attempt to access query with different user from log should fail.
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
def test_set_and_unset(self):
"""
Starts Impala cluster with a custom query option, and checks that option
overlaying works correctly.
The Beeswax API and the HiveServer2 implementations are slightly different,
so the same test is run in both contexts.
"""
# Beeswax API:
result = self.execute_query_expect_success(self.client, "set all")
assert "DEBUG_ACTION\tcustom\tDEVELOPMENT" in result.data, "baseline"
self.execute_query_expect_success(self.client, "set debug_action=hey")
assert "DEBUG_ACTION\they\tDEVELOPMENT" in \
self.execute_query_expect_success(self.client, "set all").data, "session override"
self.execute_query_expect_success(self.client, 'set debug_action=""')
assert "DEBUG_ACTION\tcustom\tDEVELOPMENT" in \
self.execute_query_expect_success(self.client, "set all").data, "reset"
self.execute_query_expect_success(self.client, 'set batch_size=123')
# Use a "request overlay" to change the option for a specific
# request within a session. We run a real query and check its
# runtime profile, as SET shows session options without applying
# the request overlays to them.
assert "BATCH_SIZE=100" in self.execute_query_expect_success(
self.client, 'select 1', query_options=dict(
batch_size="100")).runtime_profile, "request overlay"
# Overlaying an empty string (unset) has no effect; the session option
# takes hold and the "request overlay" is considered blank.
assert "BATCH_SIZE=123" in self.execute_query_expect_success(
self.client, 'select 1', query_options=dict(
batch_size="")).runtime_profile, "null request overlay"
# Same dance, but with HS2:
assert ("DEBUG_ACTION", "custom") in self.get_set_results(), "baseline"
self.execute_statement("set debug_action='hey'")
assert ("DEBUG_ACTION",
"hey") in self.get_set_results(), "session override"
self.execute_statement("set debug_action=''")
assert ("DEBUG_ACTION", "custom") in self.get_set_results(), "reset"
# Request Overlay
self.execute_statement("set batch_size=123")
execute_statement_resp = self.execute_statement(
"select 1", conf_overlay=dict(batch_size="100"))
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
assert "BATCH_SIZE=100" in self.hs2_client.GetRuntimeProfile(
get_profile_req).profile
# Null request overlay
self.execute_statement("set batch_size=999")
execute_statement_resp = self.execute_statement(
"select 1", conf_overlay=dict(batch_size=""))
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
assert "BATCH_SIZE=999" in self.hs2_client.GetRuntimeProfile(
get_profile_req).profile
def __check_hs2_query_opts(self,
pool_name,
mem_limit=None,
expected_options=None):
""" Submits a query via HS2 (optionally with a mem_limit in the confOverlay)
into pool_name and checks that the expected_query_options are set in the
profile."""
execute_statement_req = TCLIService.TExecuteStatementReq()
execute_statement_req.sessionHandle = self.session_handle
execute_statement_req.confOverlay = {'request_pool': pool_name}
if mem_limit is not None:
execute_statement_req.confOverlay['mem_limit'] = mem_limit
execute_statement_req.statement = "select 1"
execute_statement_resp = self.hs2_client.ExecuteStatement(
execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp)
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 1
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
HS2TestSuite.check_response(fetch_results_resp)
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
HS2TestSuite.check_response(
self.hs2_client.CloseOperation(close_operation_req))
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
HS2TestSuite.check_response(get_profile_resp)
self.__check_query_options(get_profile_resp.profile, expected_options)
def test_get_profile(self):
statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# If ExecuteStatement() has completed but the results haven't been fetched yet, the
# query must have at least reached RUNNING.
assert "Query State: RUNNING" in get_profile_resp.profile or \
"Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# After fetching the results, we must be in state FINISHED.
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(self.hs2_client.CloseOperation(close_operation_req))
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
def __run_stmt_and_verify_profile_access(self, stmt, has_access, close_operation):
"""Runs 'stmt' and retrieves the runtime profile and exec summary. If
'has_access' is true, it verifies that no runtime profile or exec summary are
returned. If 'close_operation' is true, make sure the operation is closed before
retrieving the profile and exec summary."""
from tests.hs2.test_hs2 import TestHS2
execute_statement_resp = self.__execute_hs2_stmt(stmt, False)
if close_operation:
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(self.hs2_client.CloseOperation(close_operation_req))
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
if has_access:
TestHS2.check_response(get_profile_resp)
assert "Plan: " in get_profile_resp.profile
else:
assert "User %s is not authorized to access the runtime profile or "\
"execution summary." % (getuser()) in str(get_profile_resp)
exec_summary_req = ImpalaHiveServer2Service.TGetExecSummaryReq()
exec_summary_req.operationHandle = execute_statement_resp.operationHandle
exec_summary_req.sessionHandle = self.session_handle
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
if has_access:
TestHS2.check_response(exec_summary_resp)
else:
assert "User %s is not authorized to access the runtime profile or "\
"execution summary." % (getuser()) in str(exec_summary_resp)
def _fetch_profile_loop(self, exception_array, query_uuid, op_handle):
try:
# This thread will keep fetching the profile to make sure that the
# ClientRequestState object can be continually accessed during unregistration.
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = op_handle
get_profile_req.sessionHandle = self.session_handle
def find_query(array):
"""Find the query for this test in a JSON array returned from web UI."""
return [q for q in array if query_uuid in q['stmt']]
# Loop until the query has been unregistered and moved out of in-flight
# queries.
registered = True
while registered:
get_profile_resp = self.hs2_client.GetRuntimeProfile(
get_profile_req)
TestHS2.check_response(get_profile_resp)
if "Unregister query" in get_profile_resp.profile:
json = self.impalad_test_service.get_queries_json()
inflight_query = find_query(json['in_flight_queries'])
completed_query = find_query(json['completed_queries'])
# Query should only be in one list.
assert len(inflight_query) + len(completed_query) == 1
if completed_query:
registered = False
except BaseException as e:
exception_array[0] = e
def __get_runtime_profile(self, op_handle):
"""Helper method to get the runtime profile from a given operation handle."""
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = op_handle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
HS2TestSuite.check_response(get_profile_resp)
return get_profile_resp.profile
def test_get_exec_summary(self):
statement = "SELECT COUNT(1) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
exec_summary_req = ImpalaHiveServer2Service.TGetExecSummaryReq()
exec_summary_req.operationHandle = execute_statement_resp.operationHandle
exec_summary_req.sessionHandle = self.session_handle
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
# Test getting the summary while query is running. We can't verify anything
# about the summary (depends how much progress query has made) but the call
# should work.
TestHS2.check_response(exec_summary_resp)
# Wait for query to start running so we can get a non-empty ExecSummary.
self.wait_for_admission_control(execute_statement_resp.operationHandle)
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
TestHS2.check_response(exec_summary_resp)
assert len(exec_summary_resp.summary.nodes) > 0
# Test that session secret is validated. Note that operation secret does not need to
# be validated in addition if the session secret is valid and the operation belongs
# to the session, because the user has full access to the session.
TestHS2.check_invalid_session(
self.hs2_client.GetExecSummary(
ImpalaHiveServer2Service.TGetExecSummaryReq(
execute_statement_resp.operationHandle,
create_session_handle_without_secret(
self.session_handle))))
# Attempt to access query with different user should fail.
evil_user = getuser() + "_evil_twin"
with ScopedSession(self.hs2_client, username=evil_user) as session:
session_handle2 = session.sessionHandle
TestHS2.check_profile_access_denied(self.hs2_client.GetExecSummary(
ImpalaHiveServer2Service.TGetExecSummaryReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
# Now close the query and verify the exec summary is available.
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(
self.hs2_client.CloseOperation(close_operation_req))
# Attempt to access query with different user from log should fail.
TestHS2.check_profile_access_denied(
self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle,
session_handle2)),
user=evil_user)
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
TestHS2.check_response(exec_summary_resp)
assert len(exec_summary_resp.summary.nodes) > 0
def get_runtime_profile(self, operation_handle, session_handle):
"""
Calls Impala HS2 API's GetRuntimeProfile method on the given query handle
:return: TExecSummary object serialized as a dict
"""
req = ImpalaHiveServer2Service.TGetRuntimeProfileReq(operationHandle=operation_handle, sessionHandle=session_handle)
# TGetRuntimeProfileReq() only works for closed queries
try:
self.close_operation(operation_handle)
except QueryServerException, e:
LOG.warn('Failed to close operation for query handle, query may be invalid or already closed.')
def test_get_profile(self):
execute_statement_req = TCLIService.TExecuteStatementReq()
execute_statement_req.sessionHandle = self.session_handle
execute_statement_req.statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
TestHS2.check_response(execute_statement_resp)
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert execute_statement_req.statement in get_profile_resp.profile
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(self.hs2_client.CloseOperation(close_operation_req))
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert execute_statement_req.statement in get_profile_resp.profile
def test_get_profile(self):
statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# If ExecuteStatement() has completed but the results haven't been fetched yet, the
# query must have reached either COMPILED or RUNNING or FINISHED.
assert "Query State: COMPILED" in get_profile_resp.profile or \
"Query State: RUNNING" in get_profile_resp.profile or \
"Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# After fetching the results, we must be in state FINISHED.
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
# Test that session secret is validated. Note that operation secret does not need to
# be validated in addition if the session secret is valid and the operation belongs
# to the session, because the user has full access to the session.
TestHS2.check_invalid_session(
self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle,
create_session_handle_without_secret(
self.session_handle))))
# Attempt to access query with different user should fail.
evil_user = getuser() + "_evil_twin"
session_handle2 = self._open_extra_session(evil_user)
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(
self.hs2_client.CloseOperation(close_operation_req))
# Attempt to access query with different user from log should fail.
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
# Check get JSON format profile
get_profile_req.format = 3 # json format
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
try:
json_res = json.loads(get_profile_resp.profile)
except ValueError:
assert False, "Download JSON format query profile cannot be parsed." \
"Response text:{0}".format(get_profile_resp.profile)
# The query statement should exist in json info_strings
if ("child_profiles" not in json_res["contents"]) or \
("info_strings" not in json_res["contents"]["child_profiles"][0]):
assert False, "JSON content is invalid. Content: {0}"\
.format(get_profile_resp.profile)
for info_string in json_res["contents"]["child_profiles"][0][
"info_strings"]:
if info_string["key"] == "Sql Statement":
assert statement in info_string["value"], \
"JSON content is invalid. Content: {0}".format(get_profile_resp.profile)
break