def test_get_tables(self, cluster_properties, unique_database): """Basic test for the GetTables() HS2 method. Needs to execute serially because the test depends on controlling whether a table is loaded or not and other concurrent tests loading or invalidating tables could interfere with it.""" table = "__hs2_column_comments_test" self.execute_query("use {0}".format(unique_database)) self.execute_query("drop table if exists {0}".format(table)) self.execute_query(""" create table {0} (a int comment 'column comment') comment 'table comment'""".format(table)) try: req = TCLIService.TGetTablesReq() req.sessionHandle = self.session_handle req.schemaName = unique_database req.tableName = table # Execute the request twice, the first time with the table unloaded and the second # with it loaded. self.execute_query("invalidate metadata {0}".format(table)) for i in range(2): get_tables_resp = self.hs2_client.GetTables(req) TestHS2.check_response(get_tables_resp) fetch_results_resp = self._fetch_results( get_tables_resp.operationHandle, 100) results = fetch_results_resp.results table_cat = results.columns[0].stringVal.values[0] table_schema = results.columns[1].stringVal.values[0] table_name = results.columns[2].stringVal.values[0] table_type = results.columns[3].stringVal.values[0] table_remarks = results.columns[4].stringVal.values[0] assert table_cat == '' assert table_schema == unique_database assert table_name == table assert table_type == "TABLE" if i == 0: assert table_remarks == "" else: if not cluster_properties.is_catalog_v2_cluster(): assert table_remarks == "table comment" # Ensure the table is loaded for the second iteration. self.execute_query("describe {0}".format(table)) # Test that session secret is validated by this API. invalid_req = TCLIService.TGetTablesReq() invalid_req.sessionHandle = create_session_handle_without_secret( self.session_handle) invalid_req.schemaName = unique_database invalid_req.tableName = table TestHS2.check_invalid_session( self.hs2_client.GetTables(invalid_req)) finally: self.execute_query("drop table {0}".format(table))
def test_get_exec_summary(self): statement = "SELECT COUNT(1) FROM functional.alltypes" execute_statement_resp = self.execute_statement(statement) exec_summary_req = ImpalaHiveServer2Service.TGetExecSummaryReq() exec_summary_req.operationHandle = execute_statement_resp.operationHandle exec_summary_req.sessionHandle = self.session_handle exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req) # Test getting the summary while query is running. We can't verify anything # about the summary (depends how much progress query has made) but the call # should work. TestHS2.check_response(exec_summary_resp) # Wait for query to start running so we can get a non-empty ExecSummary. self.wait_for_admission_control(execute_statement_resp.operationHandle) exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req) TestHS2.check_response(exec_summary_resp) assert len(exec_summary_resp.summary.nodes) > 0 # Test that session secret is validated. Note that operation secret does not need to # be validated in addition if the session secret is valid and the operation belongs # to the session, because the user has full access to the session. TestHS2.check_invalid_session( self.hs2_client.GetExecSummary( ImpalaHiveServer2Service.TGetExecSummaryReq( execute_statement_resp.operationHandle, create_session_handle_without_secret( self.session_handle)))) # Attempt to access query with different user should fail. evil_user = getuser() + "_evil_twin" session_handle2 = self._open_extra_session(evil_user) TestHS2.check_profile_access_denied(self.hs2_client.GetExecSummary( ImpalaHiveServer2Service.TGetExecSummaryReq( execute_statement_resp.operationHandle, session_handle2)), user=evil_user) # Now close the query and verify the exec summary is available. close_operation_req = TCLIService.TCloseOperationReq() close_operation_req.operationHandle = execute_statement_resp.operationHandle TestHS2.check_response( self.hs2_client.CloseOperation(close_operation_req)) # Attempt to access query with different user from log should fail. TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile( ImpalaHiveServer2Service.TGetRuntimeProfileReq( execute_statement_resp.operationHandle, session_handle2)), user=evil_user) exec_summary_resp = self.hs2_client.GetExecSummary(exec_summary_req) TestHS2.check_response(exec_summary_resp) assert len(exec_summary_resp.summary.nodes) > 0
def test_close_session(self): """Test that an open session can be closed""" open_session_req = TCLIService.TOpenSessionReq() resp = self.hs2_client.OpenSession(open_session_req) TestHS2.check_response(resp) # Check that CloseSession validates session secret and acts as if the session didn't # exist. invalid_close_session_req = TCLIService.TCloseSessionReq() invalid_close_session_req.sessionHandle = create_session_handle_without_secret( resp.sessionHandle) TestHS2.check_invalid_session( self.hs2_client.CloseSession(invalid_close_session_req)) close_session_req = TCLIService.TCloseSessionReq() close_session_req.sessionHandle = resp.sessionHandle TestHS2.check_response(self.hs2_client.CloseSession(close_session_req))
def test_get_schemas(self): get_schemas_req = TCLIService.TGetSchemasReq() get_schemas_req.sessionHandle = self.session_handle get_schemas_resp = self.hs2_client.GetSchemas(get_schemas_req) TestHS2.check_response(get_schemas_resp) fetch_results_req = TCLIService.TFetchResultsReq() fetch_results_req.operationHandle = get_schemas_resp.operationHandle fetch_results_req.maxRows = 100 fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req) TestHS2.check_response(fetch_results_resp) query_id = operation_id_to_query_id( get_schemas_resp.operationHandle.operationId) profile_page = self.impalad_test_service.read_query_profile_page( query_id) # Test fix for IMPALA-619 assert "Sql Statement: GET_SCHEMAS" in profile_page assert "Query Type: DDL" in profile_page # Test that session secret is validated by this API. get_schemas_req.sessionHandle = create_session_handle_without_secret( self.session_handle) TestHS2.check_invalid_session( self.hs2_client.GetSchemas(get_schemas_req))
def test_get_profile(self): statement = "SELECT COUNT(2) FROM functional.alltypes" execute_statement_resp = self.execute_statement(statement) get_profile_req = ImpalaHiveServer2Service.TGetRuntimeProfileReq() get_profile_req.operationHandle = execute_statement_resp.operationHandle get_profile_req.sessionHandle = self.session_handle get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req) TestHS2.check_response(get_profile_resp) assert statement in get_profile_resp.profile # If ExecuteStatement() has completed but the results haven't been fetched yet, the # query must have reached either COMPILED or RUNNING or FINISHED. assert "Query State: COMPILED" in get_profile_resp.profile or \ "Query State: RUNNING" in get_profile_resp.profile or \ "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile fetch_results_req = TCLIService.TFetchResultsReq() fetch_results_req.operationHandle = execute_statement_resp.operationHandle fetch_results_req.maxRows = 100 fetch_results_resp = self.hs2_client.FetchResults(fetch_results_req) get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req) TestHS2.check_response(get_profile_resp) assert statement in get_profile_resp.profile # After fetching the results, we must be in state FINISHED. assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile # Test that session secret is validated. Note that operation secret does not need to # be validated in addition if the session secret is valid and the operation belongs # to the session, because the user has full access to the session. TestHS2.check_invalid_session( self.hs2_client.GetRuntimeProfile( ImpalaHiveServer2Service.TGetRuntimeProfileReq( execute_statement_resp.operationHandle, create_session_handle_without_secret( self.session_handle)))) # Attempt to access query with different user should fail. evil_user = getuser() + "_evil_twin" session_handle2 = self._open_extra_session(evil_user) TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile( ImpalaHiveServer2Service.TGetRuntimeProfileReq( execute_statement_resp.operationHandle, session_handle2)), user=evil_user) close_operation_req = TCLIService.TCloseOperationReq() close_operation_req.operationHandle = execute_statement_resp.operationHandle TestHS2.check_response( self.hs2_client.CloseOperation(close_operation_req)) # Attempt to access query with different user from log should fail. TestHS2.check_profile_access_denied(self.hs2_client.GetRuntimeProfile( ImpalaHiveServer2Service.TGetRuntimeProfileReq( execute_statement_resp.operationHandle, session_handle2)), user=evil_user) get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req) TestHS2.check_response(get_profile_resp) assert statement in get_profile_resp.profile assert "Query State: FINISHED" in get_profile_resp.profile, get_profile_resp.profile # Check get JSON format profile get_profile_req.format = 3 # json format get_profile_resp = self.hs2_client.GetRuntimeProfile(get_profile_req) TestHS2.check_response(get_profile_resp) try: json_res = json.loads(get_profile_resp.profile) except ValueError: assert False, "Download JSON format query profile cannot be parsed." \ "Response text:{0}".format(get_profile_resp.profile) # The query statement should exist in json info_strings if ("child_profiles" not in json_res["contents"]) or \ ("info_strings" not in json_res["contents"]["child_profiles"][0]): assert False, "JSON content is invalid. Content: {0}"\ .format(get_profile_resp.profile) for info_string in json_res["contents"]["child_profiles"][0][ "info_strings"]: if info_string["key"] == "Sql Statement": assert statement in info_string["value"], \ "JSON content is invalid. Content: {0}".format(get_profile_resp.profile) break
def test_get_info(self): # Negative test for invalid session secret. invalid_req = TCLIService.TGetInfoReq( create_session_handle_without_secret(self.session_handle), TCLIService.TGetInfoType.CLI_DBMS_NAME) TestHS2.check_invalid_session(self.hs2_client.GetInfo(invalid_req))
def test_get_tables(self, cluster_properties, unique_database): """Basic test for the GetTables() HS2 method. Needs to execute serially because the test depends on controlling whether a table is loaded or not and other concurrent tests loading or invalidating tables could interfere with it.""" table = "__hs2_table_comment_test" view = "__hs2_view_comment_test" self.execute_query("use {0}".format(unique_database)) self.execute_query("drop table if exists {0}".format(table)) self.execute_query("drop view if exists {0}".format(view)) self.execute_query(""" create table {0} (a int comment 'column comment') comment 'table comment'""".format(table)) self.execute_query(""" create view {0} comment 'view comment' as select * from {1} """.format(view, table)) try: req = TCLIService.TGetTablesReq() req.sessionHandle = self.session_handle req.schemaName = unique_database # Execute the request twice, the first time with the table/view unloaded and the # second with them loaded. self.execute_query("invalidate metadata {0}".format(table)) self.execute_query("invalidate metadata {0}".format(view)) for i in range(2): get_tables_resp = self.hs2_client.GetTables(req) TestHS2.check_response(get_tables_resp) fetch_results_resp = self._fetch_results( get_tables_resp.operationHandle, 100) results = fetch_results_resp.results # The returned results should only contain metadata for the table and the view. # Note that results are in columnar format so we get #rows by the length of the # first column. assert len(results.columns[0].stringVal.values) == 2 for row_idx in range(2): table_cat = results.columns[0].stringVal.values[row_idx] table_schema = results.columns[1].stringVal.values[row_idx] table_name = results.columns[2].stringVal.values[row_idx] table_type = results.columns[3].stringVal.values[row_idx] table_remarks = results.columns[4].stringVal.values[ row_idx] assert table_cat == '' assert table_schema == unique_database if i == 0: # In the first iteration the table and view are unloaded, so their types are # in the default value (TABLE) and their comments are empty. assert table_name in (table, view) assert table_type == "TABLE" assert table_remarks == "" else: # In the second iteration the table and view are loaded. They should be shown # with correct types and comments. Verify types and comments based on table # names so we don't care the order in which the table/view are returned. if table_name == table: assert table_type == "TABLE" assert table_remarks == "table comment" else: assert table_name == view assert table_type == "VIEW" assert table_remarks == "view comment" # Ensure the table and view are loaded for the second iteration. self.execute_query("describe {0}".format(table)) self.execute_query("describe {0}".format(view)) # Test that session secret is validated by this API. invalid_req = TCLIService.TGetTablesReq() invalid_req.sessionHandle = create_session_handle_without_secret( self.session_handle) invalid_req.schemaName = unique_database invalid_req.tableName = table TestHS2.check_invalid_session( self.hs2_client.GetTables(invalid_req)) finally: self.execute_query("drop table {0}".format(table))