def test_get_tables(self, cluster_properties, unique_database):
"""Basic test for the GetTables() HS2 method. Needs to execute serially because
the test depends on controlling whether a table is loaded or not and other
concurrent tests loading or invalidating tables could interfere with it."""
table = "__hs2_column_comments_test"
self.execute_query("use {0}".format(unique_database))
self.execute_query("drop table if exists {0}".format(table))
self.execute_query("""
create table {0} (a int comment 'column comment')
comment 'table comment'""".format(table))
try:
req = TCLIService.TGetTablesReq()
req.sessionHandle = self.session_handle
req.schemaName = unique_database
req.tableName = table
# Execute the request twice, the first time with the table unloaded and the second
# with it loaded.
self.execute_query("invalidate metadata {0}".format(table))
for i in range(2):
get_tables_resp = self.hs2_client.GetTables(req)
TestHS2.check_response(get_tables_resp)
fetch_results_resp = self._fetch_results(
get_tables_resp.operationHandle, 100)
results = fetch_results_resp.results
table_cat = results.columns[0].stringVal.values[0]
table_schema = results.columns[1].stringVal.values[0]
table_name = results.columns[2].stringVal.values[0]
table_type = results.columns[3].stringVal.values[0]
table_remarks = results.columns[4].stringVal.values[0]
assert table_cat == ''
assert table_schema == unique_database
assert table_name == table
assert table_type == "TABLE"
if i == 0:
assert table_remarks == ""
else:
if not cluster_properties.is_catalog_v2_cluster():
assert table_remarks == "table comment"
# Ensure the table is loaded for the second iteration.
self.execute_query("describe {0}".format(table))
# Test that session secret is validated by this API.
invalid_req = TCLIService.TGetTablesReq()
invalid_req.sessionHandle = create_session_handle_without_secret(
self.session_handle)
invalid_req.schemaName = unique_database
invalid_req.tableName = table
TestHS2.check_invalid_session(
self.hs2_client.GetTables(invalid_req))
finally:
self.execute_query("drop table {0}".format(table))
def test_get_exec_summary(self):
statement = "SELECT COUNT(1) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
exec_summary_req = ImpalaHiveServer2Service.TGetExecSummaryReq()
exec_summary_req.operationHandle = execute_statement_resp.operationHandle
exec_summary_req.sessionHandle = self.session_handle
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
# Test getting the summary while query is running. We can't verify anything
# about the summary (depends how much progress query has made) but the call
# should work.
TestHS2.check_response(exec_summary_resp)
# Wait for query to start running so we can get a non-empty ExecSummary.
self.wait_for_admission_control(execute_statement_resp.operationHandle)
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
TestHS2.check_response(exec_summary_resp)
assert len(exec_summary_resp.summary.nodes) > 0
# Test that session secret is validated. Note that operation secret does not need to
# be validated in addition if the session secret is valid and the operation belongs
# to the session, because the user has full access to the session.
TestHS2.check_invalid_session(
self.hs2_client.GetExecSummary(
ImpalaHiveServer2Service.TGetExecSummaryReq(
execute_statement_resp.operationHandle,
create_session_handle_without_secret(
self.session_handle))))
# Attempt to access query with different user should fail.
evil_user = getuser() + "_evil_twin"
session_handle2 = self._open_extra_session(evil_user)
TestHS2.check_profile_access_denied(self.hs2_client.GetExecSummary(
ImpalaHiveServer2Service.TGetExecSummaryReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
# Now close the query and verify the exec summary is available.
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(
self.hs2_client.CloseOperation(close_operation_req))
# Attempt to access query with different user from log should fail.
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req)
TestHS2.check_response(exec_summary_resp)
assert len(exec_summary_resp.summary.nodes) > 0
def test_close_session(self):
"""Test that an open session can be closed"""
open_session_req = TCLIService.TOpenSessionReq()
resp = self.hs2_client.OpenSession(open_session_req)
TestHS2.check_response(resp)
# Check that CloseSession validates session secret and acts as if the session didn't
# exist.
invalid_close_session_req = TCLIService.TCloseSessionReq()
invalid_close_session_req.sessionHandle = create_session_handle_without_secret(
resp.sessionHandle)
TestHS2.check_invalid_session(
self.hs2_client.CloseSession(invalid_close_session_req))
close_session_req = TCLIService.TCloseSessionReq()
close_session_req.sessionHandle = resp.sessionHandle
TestHS2.check_response(self.hs2_client.CloseSession(close_session_req))
def test_get_schemas(self):
get_schemas_req = TCLIService.TGetSchemasReq()
get_schemas_req.sessionHandle = self.session_handle
get_schemas_resp = self.hs2_client.GetSchemas(get_schemas_req)
TestHS2.check_response(get_schemas_resp)
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = get_schemas_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
TestHS2.check_response(fetch_results_resp)
query_id = operation_id_to_query_id(
get_schemas_resp.operationHandle.operationId)
profile_page = self.impalad_test_service.read_query_profile_page(
query_id)
# Test fix for IMPALA-619
assert "Sql Statement: GET_SCHEMAS" in profile_page
assert "Query Type: DDL" in profile_page
# Test that session secret is validated by this API.
get_schemas_req.sessionHandle = create_session_handle_without_secret(
self.session_handle)
TestHS2.check_invalid_session(
self.hs2_client.GetSchemas(get_schemas_req))
def test_get_profile(self):
statement = "SELECT COUNT(2) FROM functional.alltypes"
execute_statement_resp = self.execute_statement(statement)
get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq()
get_profile_req.operationHandle = execute_statement_resp.operationHandle
get_profile_req.sessionHandle = self.session_handle
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# If ExecuteStatement() has completed but the results haven't been fetched yet, the
# query must have reached either COMPILED or RUNNING or FINISHED.
assert "Query State: COMPILED" in get_profile_resp.profile or \
"Query State: RUNNING" in get_profile_resp.profile or \
"Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
fetch_results_req = TCLIService.TFetchResultsReq()
fetch_results_req.operationHandle = execute_statement_resp.operationHandle
fetch_results_req.maxRows = 100
fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
# After fetching the results, we must be in state FINISHED.
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
# Test that session secret is validated. Note that operation secret does not need to
# be validated in addition if the session secret is valid and the operation belongs
# to the session, because the user has full access to the session.
TestHS2.check_invalid_session(
self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle,
create_session_handle_without_secret(
self.session_handle))))
# Attempt to access query with different user should fail.
evil_user = getuser() + "_evil_twin"
session_handle2 = self._open_extra_session(evil_user)
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
close_operation_req = TCLIService.TCloseOperationReq()
close_operation_req.operationHandle = execute_statement_resp.operationHandle
TestHS2.check_response(
self.hs2_client.CloseOperation(close_operation_req))
# Attempt to access query with different user from log should fail.
TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile(
ImpalaHiveServer2Service.TGetRuntimeProfileReq(
execute_statement_resp.operationHandle, session_handle2)),
user=evil_user)
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
assert statement in get_profile_resp.profile
assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile
# Check get JSON format profile
get_profile_req.format = 3 # json format
get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req)
TestHS2.check_response(get_profile_resp)
try:
json_res = json.loads(get_profile_resp.profile)
except ValueError:
assert False, "Download JSON format query profile cannot be parsed." \
"Response text:{0}".format(get_profile_resp.profile)
# The query statement should exist in json info_strings
if ("child_profiles" not in json_res["contents"]) or \
("info_strings" not in json_res["contents"]["child_profiles"][0]):
assert False, "JSON content is invalid. Content: {0}"\
.format(get_profile_resp.profile)
for info_string in json_res["contents"]["child_profiles"][0][
"info_strings"]:
if info_string["key"] == "Sql Statement":
assert statement in info_string["value"], \
"JSON content is invalid. Content: {0}".format(get_profile_resp.profile)
break
def test_get_info(self):
# Negative test for invalid session secret.
invalid_req = TCLIService.TGetInfoReq(
create_session_handle_without_secret(self.session_handle),
TCLIService.TGetInfoType.CLI_DBMS_NAME)
TestHS2.check_invalid_session(self.hs2_client.GetInfo(invalid_req))
def test_get_tables(self, cluster_properties, unique_database):
"""Basic test for the GetTables() HS2 method. Needs to execute serially because
the test depends on controlling whether a table is loaded or not and other
concurrent tests loading or invalidating tables could interfere with it."""
table = "__hs2_table_comment_test"
view = "__hs2_view_comment_test"
self.execute_query("use {0}".format(unique_database))
self.execute_query("drop table if exists {0}".format(table))
self.execute_query("drop view if exists {0}".format(view))
self.execute_query("""
create table {0} (a int comment 'column comment')
comment 'table comment'""".format(table))
self.execute_query("""
create view {0} comment 'view comment' as select * from {1}
""".format(view, table))
try:
req = TCLIService.TGetTablesReq()
req.sessionHandle = self.session_handle
req.schemaName = unique_database
# Execute the request twice, the first time with the table/view unloaded and the
# second with them loaded.
self.execute_query("invalidate metadata {0}".format(table))
self.execute_query("invalidate metadata {0}".format(view))
for i in range(2):
get_tables_resp = self.hs2_client.GetTables(req)
TestHS2.check_response(get_tables_resp)
fetch_results_resp = self._fetch_results(
get_tables_resp.operationHandle, 100)
results = fetch_results_resp.results
# The returned results should only contain metadata for the table and the view.
# Note that results are in columnar format so we get #rows by the length of the
# first column.
assert len(results.columns[0].stringVal.values) == 2
for row_idx in range(2):
table_cat = results.columns[0].stringVal.values[row_idx]
table_schema = results.columns[1].stringVal.values[row_idx]
table_name = results.columns[2].stringVal.values[row_idx]
table_type = results.columns[3].stringVal.values[row_idx]
table_remarks = results.columns[4].stringVal.values[
row_idx]
assert table_cat == ''
assert table_schema == unique_database
if i == 0:
# In the first iteration the table and view are unloaded, so their types are
# in the default value (TABLE) and their comments are empty.
assert table_name in (table, view)
assert table_type == "TABLE"
assert table_remarks == ""
else:
# In the second iteration the table and view are loaded. They should be shown
# with correct types and comments. Verify types and comments based on table
# names so we don't care the order in which the table/view are returned.
if table_name == table:
assert table_type == "TABLE"
assert table_remarks == "table comment"
else:
assert table_name == view
assert table_type == "VIEW"
assert table_remarks == "view comment"
# Ensure the table and view are loaded for the second iteration.
self.execute_query("describe {0}".format(table))
self.execute_query("describe {0}".format(view))
# Test that session secret is validated by this API.
invalid_req = TCLIService.TGetTablesReq()
invalid_req.sessionHandle = create_session_handle_without_secret(
self.session_handle)
invalid_req.schemaName = unique_database
invalid_req.tableName = table
TestHS2.check_invalid_session(
self.hs2_client.GetTables(invalid_req))
finally:
self.execute_query("drop table {0}".format(table))
