def test_exchange_eos(self, vector):
""" Test IMPALA-8845: runs with result spooling enabled and defers the fetching
of results until all non-coordinator fragments have completed. It aims to verify
that once the coordinator fragment reaches eos, the rest of the fragments will
get unblocked. Using a cluster of size 9 which can reliably reproduce the hang of
some non-coordinator fragments without the fix of IMPALA-8845.
"""
cluster = ImpalaCluster.get_e2e_test_cluster()
coordinator = cluster.get_first_impalad()
client = coordinator.service.create_beeswax_client()
vector.get_value('exec_option')['spool_query_results'] = 'true'
for query in [
"select * from tpch.lineitem order by l_orderkey limit 10000",
"select * from tpch.lineitem limit 10000"
]:
handle = self.execute_query_async_using_client(
client, query, vector)
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
if impalad.get_webserver_port(
) == coordinator.get_webserver_port():
num_fragments = 1
else:
num_fragments = 0
verifier.wait_for_metric(
"impala-server.num-fragments-in-flight", num_fragments)
results = client.fetch(query, handle)
assert results.success
assert len(results.data) == 10000
client.close()
def execute(self, query):
"""Executes a query on the coordinator of the local minicluster."""
cluster = ImpalaCluster.get_e2e_test_cluster()
if len(cluster.impalads) == 0:
raise Exception("Coordinator not running")
client = cluster.get_first_impalad().service.create_hs2_client()
return client.execute(query)
def test_run_invalidate_refresh(self, vector):
"""Verifies that running concurrent invalidate table/catalog and refresh commands
don't cause failures with other running workloads and ensures catalog versions
are strictly increasing."""
target_db = self.execute_scalar('select current_database()', vector=vector)
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = impala_cluster.impalads[0].service
catalogd = impala_cluster.catalogd.service
for i in xrange(NUM_ITERATIONS):
# Get the catalog versions for the table before running the workload
before_versions = dict()
before_versions['catalogd'] =\
self.get_table_version(catalogd, target_db, 'lineitem')
before_versions['impalad'] = self.get_table_version(impalad, target_db, 'lineitem')
self.run_test_case('stress-with-invalidate-refresh', vector)
# Get the catalog versions for the table after running the workload
after_versions = dict()
after_versions['catalogd'] = self.get_table_version(catalogd, target_db, 'lineitem')
after_versions['impalad'] = self.get_table_version(impalad, target_db, 'lineitem')
# Catalog versions should be strictly increasing
assert before_versions['impalad'] < after_versions['impalad']
assert before_versions['catalogd'] < after_versions['catalogd']
def test_insert_mem_limit(self, vector, unique_database):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
vector.get_value('compression_codec')
self.run_test_case(
'QueryTest/insert-mem-limit',
vector,
unique_database,
multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1,
test_file_vars={
'$ORIGINAL_DB':
ImpalaTestSuite.get_db_name_from_format(
vector.get_value('table_format'))
})
# IMPALA-7023: These queries can linger and use up memory, causing subsequent
# tests to hit memory limits. Wait for some time to allow the query to
# be reclaimed.
verifiers = [
MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads
]
for v in verifiers:
v.wait_for_metric("impala-server.num-fragments-in-flight",
0,
timeout=180)
def test_low_mem_limit_no_fragments(self, vector):
self.low_memory_limit_test(vector, 'tpch-q14', self.MIN_MEM_FOR_TPCH['Q14'])
self.low_memory_limit_test(vector, 'tpch-q18', self.MIN_MEM_FOR_TPCH['Q18'])
self.low_memory_limit_test(vector, 'tpch-q20', self.MIN_MEM_FOR_TPCH['Q20'])
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0)
def test_failpoints(self, vector):
query = vector.get_value('query')
action = vector.get_value('action')
location = vector.get_value('location')
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
plan_node_ids = self.__parse_plan_nodes_from_explain(query, vector)
for node_id in plan_node_ids:
debug_action = '%d:%s:%s' % (node_id, location,
FAILPOINT_ACTION_MAP[action])
# IMPALA-7046: add jitter to backend startup to exercise various failure paths.
debug_action += '|COORD_BEFORE_EXEC_RPC:JITTER@[email protected]'
LOG.info('Current debug action: SET DEBUG_ACTION=%s' %
debug_action)
vector.get_value('exec_option')['debug_action'] = debug_action
if action == 'CANCEL':
self.__execute_cancel_action(query, vector)
elif action == 'FAIL' or action == 'MEM_LIMIT_EXCEEDED':
self.__execute_fail_action(query, vector)
else:
assert 0, 'Unknown action: %s' % action
# We should be able to execute the same query successfully when no failures are
# injected.
del vector.get_value('exec_option')['debug_action']
self.execute_query(query, vector.get_value('exec_option'))
# Detect any hung fragments left from this test.
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight",
0)
def _verify_no_fragments_running(self):
"""Raise an exception if there are fragments running on the cluster after a
timeout."""
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0, timeout=10)
verifier.wait_for_backend_admission_control_state(timeout=10)
def __test_invalid_result_caching(self, sql_stmt):
""" Tests that invalid requests for query-result caching fail
using the given sql_stmt."""
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = impala_cluster.impalads[0].service
execute_statement_req = TCLIService.TExecuteStatementReq()
execute_statement_req.sessionHandle = self.session_handle
execute_statement_req.statement = sql_stmt
execute_statement_req.confOverlay = dict()
# Test that a malformed result-cache size returns an error.
execute_statement_req.confOverlay[
self.IMPALA_RESULT_CACHING_OPT] = "bad_number"
execute_statement_resp = self.hs2_client.ExecuteStatement(
execute_statement_req)
HS2TestSuite.check_response(
execute_statement_resp, TCLIService.TStatusCode.ERROR_STATUS,
"Invalid value 'bad_number' for 'impala.resultset.cache.size' option"
)
self.__verify_num_cached_rows(0)
assert 0 == impalad.get_num_in_flight_queries()
# Test that a result-cache size exceeding the per-Impalad maximum returns an error.
# The default maximum result-cache size is 100000.
execute_statement_req.confOverlay[
self.IMPALA_RESULT_CACHING_OPT] = "100001"
execute_statement_resp = self.hs2_client.ExecuteStatement(
execute_statement_req)
HS2TestSuite.check_response(
execute_statement_resp, TCLIService.TStatusCode.ERROR_STATUS,
"Requested result-cache size of 100001 exceeds Impala's maximum of 100000"
)
self.__verify_num_cached_rows(0)
assert 0 == impalad.get_num_in_flight_queries()
def _verify_describe_view(self, vector, view_name, expected_substr):
"""
Verify across all impalads that the view 'view_name' has the given substring in its
expanded SQL.
If SYNC_DDL is enabled, the verification should complete immediately. Otherwise,
loops waiting for the expected condition to pass.
"""
if vector.get_value('exec_option')['sync_ddl']:
num_attempts = 1
else:
num_attempts = 60
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
client = impalad.service.create_beeswax_client()
try:
for attempt in itertools.count(1):
assert attempt <= num_attempts, "ran out of attempts"
try:
result = self.execute_query_expect_success(
client, "describe formatted %s" % view_name)
exp_line = [l for l in result.data if 'View Expanded' in l][0]
except ImpalaBeeswaxException, e:
# In non-SYNC_DDL tests, it's OK to get a "missing view" type error
# until the metadata propagates.
exp_line = "Exception: %s" % e
if expected_substr in exp_line.lower():
return
time.sleep(1)
finally:
client.close()
def test_jvm_mem_tracking(self, vector):
service = ImpalaCluster.get_e2e_test_cluster().impalads[0].service
verifier = MemUsageVerifier(service)
proc_values = verifier.get_mem_usage_values('Process')
proc_total = proc_values['total']
proc_limit = proc_values['limit']
max_heap_size = verifier.get_mem_usage_values('JVM: max heap size')['total']
non_heap_committed = verifier.get_mem_usage_values('JVM: non-heap committed')['total']
MB = 1024 * 1024
LOG.info("proc_total={0}, max_heap_size={1} non_heap_committed={2}".format(
proc_total, max_heap_size, non_heap_committed))
# The max heap size will be lower than -Xmx but should be in the same general range.
assert max_heap_size >= 900 * MB and max_heap_size <= 1024 * MB
# The non-heap committed value is hard to predict but should be non-zero.
assert non_heap_committed > 0
# Process mem consumption should include both of the above values.
assert proc_total > max_heap_size + non_heap_committed
# Make sure that the admittable memory is within 100MB of the process limit
# minus the heap size (there may be some rounding errors).
backend_json = json.loads(service.read_debug_webpage('backends?json'))
admit_limit_human_readable = backend_json['backends'][0]['admit_mem_limit']
admit_limit = parse_mem_value(admit_limit_human_readable)
LOG.info("proc_limit={0}, admit_limit={1}".format(proc_limit, admit_limit))
assert abs(admit_limit - (proc_limit - max_heap_size)) <= 100 * MB
def test_jvm_mem_tracking(self, vector):
service = ImpalaCluster.get_e2e_test_cluster().impalads[0].service
verifier = MemUsageVerifier(service)
proc_values = verifier.get_mem_usage_values('Process')
proc_total = proc_values['total']
proc_limit = proc_values['limit']
max_heap_size = verifier.get_mem_usage_values(
'JVM: max heap size')['total']
non_heap_committed = verifier.get_mem_usage_values(
'JVM: non-heap committed')['total']
MB = 1024 * 1024
LOG.info(
"proc_total={0}, max_heap_size={1} non_heap_committed={2}".format(
proc_total, max_heap_size, non_heap_committed))
# The max heap size will be lower than -Xmx but should be in the same general range.
assert max_heap_size >= 900 * MB and max_heap_size <= 1024 * MB
# The non-heap committed value is hard to predict but should be non-zero.
assert non_heap_committed > 0
# Process mem consumption should include both of the above values.
assert proc_total > max_heap_size + non_heap_committed
# Make sure that the admittable memory is within 100MB of the process limit
# minus the heap size (there may be some rounding errors).
backend_json = json.loads(service.read_debug_webpage('backends?json'))
admit_limit_human_readable = backend_json['backends'][0][
'admit_mem_limit']
admit_limit = parse_mem_value(admit_limit_human_readable)
LOG.info("proc_limit={0}, admit_limit={1}".format(
proc_limit, admit_limit))
assert abs(admit_limit - (proc_limit - max_heap_size)) <= 100 * MB
def test_views_describe(self, vector, unique_database):
# IMPALA-6896: Tests that altered views can be described by all impalads.
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
impalads = impala_cluster.impalads
view_name = "%s.test_describe_view" % unique_database
query_opts = vector.get_value('exec_option')
first_client = impalads[0].service.create_beeswax_client()
try:
# Create a view and verify it's visible.
self.execute_query_expect_success(
first_client, "create view {0} as "
"select * from functional.alltypes".format(view_name),
query_opts)
self._verify_describe_view(vector, view_name,
"select * from functional.alltypes")
# Alter the view and verify the alter is visible.
self.execute_query_expect_success(
first_client, "alter view {0} as "
"select * from functional.alltypesagg".format(view_name),
query_opts)
self._verify_describe_view(vector, view_name,
"select * from functional.alltypesagg")
finally:
first_client.close()
def test_failure_in_prepare_multi_fragment(self):
# Test that if one fragment fails that the others are cleaned up during the ensuing
# cancellation.
verifiers = [
MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads
]
# Fail the scan node
self.client.execute("SET DEBUG_ACTION='-1:0:PREPARE:FAIL'")
# Force a query plan that will have three fragments or more.
try:
self.client.execute(
"SELECT COUNT(*) FROM functional.alltypes a JOIN [SHUFFLE] \
functional.alltypes b on a.id = b.id")
assert "Query should have thrown an error"
except ImpalaBeeswaxException:
pass
for v in verifiers:
# Long timeout required because fragments may be blocked while sending data. The
# default value of --datastream_sender_timeout_ms is 120s before they wake up and
# cancel themselves.
#
# TODO: Fix when we have cancellable RPCs.
v.wait_for_metric(self.IN_FLIGHT_FRAGMENTS, 0, timeout=125)
def test_backends_are_idle(self):
"""Test that the backends state is in a valid state when quiesced - i.e.
no queries are running and the admission control state reflects that no
resources are used."""
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_backend_admission_control_state()
def _run_query_all_impalads(self, exec_options, query, expected):
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
for impalad in impala_cluster.impalads:
client = impalad.service.create_beeswax_client()
result = self.execute_query_expect_success(client, query,
exec_options)
assert result.data == expected, impalad
def __test_invalid_result_caching(self, sql_stmt):
""" Tests that invalid requests for query-result caching fail
using the given sql_stmt."""
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = impala_cluster.impalads[0].service
execute_statement_req = TCLIService.TExecuteStatementReq()
execute_statement_req.sessionHandle = self.session_handle
execute_statement_req.statement = sql_stmt
execute_statement_req.confOverlay = dict()
# Test that a malformed result-cache size returns an error.
execute_statement_req.confOverlay[self.IMPALA_RESULT_CACHING_OPT] = "bad_number"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp,
TCLIService.TStatusCode.ERROR_STATUS,
"Invalid value 'bad_number' for 'impala.resultset.cache.size' option")
self.__verify_num_cached_rows(0)
assert 0 == impalad.get_num_in_flight_queries()
# Test that a result-cache size exceeding the per-Impalad maximum returns an error.
# The default maximum result-cache size is 100000.
execute_statement_req.confOverlay[self.IMPALA_RESULT_CACHING_OPT] = "100001"
execute_statement_resp = self.hs2_client.ExecuteStatement(execute_statement_req)
HS2TestSuite.check_response(execute_statement_resp,
TCLIService.TStatusCode.ERROR_STATUS,
"Requested result-cache size of 100001 exceeds Impala's maximum of 100000")
self.__verify_num_cached_rows(0)
assert 0 == impalad.get_num_in_flight_queries()
def test_hash_join_timer(self, vector):
# This test runs serially because it requires the query to come back within
# some amount of time. Running this with other tests makes it hard to bound
# that time. It also assumes that it will be able to get a thread token to
# execute the join build in parallel.
test_case = vector.get_value('test cases')
query = test_case[0]
join_type = test_case[1]
# Ensure that the cluster is idle before starting the test query.
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight",
0)
# Execute the query. The query summary and profile are stored in 'result'.
result = self.execute_query(query, vector.get_value('exec_option'))
# Parse the query summary; The join node is "id=3".
# In the ExecSummary, search for the join operator's summary and verify the
# avg and max times are within acceptable limits.
exec_summary = result.exec_summary
check_execsummary_count = 0
join_node_name = "03:%s" % (join_type)
for line in exec_summary:
if line['operator'] == join_node_name:
avg_time_ms = line['avg_time'] / self.NANOS_PER_MILLI
self.__verify_join_time(avg_time_ms, "ExecSummary Avg")
max_time_ms = line['max_time'] / self.NANOS_PER_MILLI
self.__verify_join_time(max_time_ms, "ExecSummary Max")
check_execsummary_count += 1
assert (check_execsummary_count == 1), \
"Unable to verify ExecSummary: {0}".format(exec_summary)
# Parse the query profile; The join node is "id=3".
# In the profiles, search for lines containing "(id=3)" and parse for the avg and
# non-child times to verify that they are within acceptable limits. Also verify
# that the build side is built in a different thread by searching for the string:
# "Join Build-Side Prepared Asynchronously"
profile = result.runtime_profile
check_fragment_count = 0
asyn_build = False
for line in profile.split("\n"):
if ("(id=3)" in line):
# Sample line:
# HASH_JOIN_NODE (id=3):(Total: 3s580ms, non-child: 11.89ms, % non-child: 0.31%)
strip1 = re.split("non-child: ", line)[1]
non_child_time = re.split(", ", strip1)[0]
non_child_time_ms = parse_duration_string_ms(non_child_time)
self.__verify_join_time(non_child_time_ms,
"Fragment non-child")
check_fragment_count += 1
# Search for "Join Build-Side Prepared Asynchronously"
if ("Join Build-Side Prepared Asynchronously" in line):
asyn_build = True
assert (asyn_build
), "Join is not prepared asynchronously: {0}".format(profile)
assert (check_fragment_count > 1), \
"Unable to verify Fragment or Average Fragment: {0}".format(profile)
def test_query_profile_encoded_unknown_query_id(self):
"""Test that /query_profile_encoded error message starts with the expected line in
case of missing query and does not contain any leading whitespace.
"""
cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = cluster.get_any_impalad()
result = impalad.service.read_debug_webpage("query_profile_encoded?query_id=123")
assert result.startswith("Could not obtain runtime profile: Query id")
def test_query_profile_encoded_unknown_query_id(self):
"""Test that /query_profile_encoded error message starts with the expected line in
case of missing query and does not contain any leading whitespace.
"""
cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = cluster.get_any_impalad()
result = impalad.service.read_debug_webpage("query_profile_encoded?query_id=123")
assert result.startswith("Could not obtain runtime profile: Query id")
def test_hash_join_timer(self, vector):
# This test runs serially because it requires the query to come back within
# some amount of time. Running this with other tests makes it hard to bound
# that time. It also assumes that it will be able to get a thread token to
# execute the join build in parallel.
test_case = vector.get_value('test cases')
query = test_case[0]
join_type = test_case[1]
# Ensure that the cluster is idle before starting the test query.
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0)
# Execute the query. The query summary and profile are stored in 'result'.
result = self.execute_query(query, vector.get_value('exec_option'))
# Parse the query summary; The join node is "id=3".
# In the ExecSummary, search for the join operator's summary and verify the
# avg and max times are within acceptable limits.
exec_summary = result.exec_summary
check_execsummary_count = 0
join_node_name = "03:%s" % (join_type)
for line in exec_summary:
if line['operator'] == join_node_name:
avg_time_ms = line['avg_time'] / self.NANOS_PER_MILLI
self.__verify_join_time(avg_time_ms, "ExecSummary Avg")
max_time_ms = line['max_time'] / self.NANOS_PER_MILLI
self.__verify_join_time(max_time_ms, "ExecSummary Max")
check_execsummary_count += 1
assert (check_execsummary_count == 1), \
"Unable to verify ExecSummary: {0}".format(exec_summary)
# Parse the query profile; The join node is "id=3".
# In the profiles, search for lines containing "(id=3)" and parse for the avg and
# non-child times to verify that they are within acceptable limits. Also verify
# that the build side is built in a different thread by searching for the string:
# "Join Build-Side Prepared Asynchronously"
profile = result.runtime_profile
check_fragment_count = 0
asyn_build = False
for line in profile.split("\n"):
if ("(id=3)" in line):
# Sample line:
# HASH_JOIN_NODE (id=3):(Total: 3s580ms, non-child: 11.89ms, % non-child: 0.31%)
strip1 = re.split("non-child: ", line)[1]
non_child_time = re.split(", ", strip1)[0]
non_child_time_ms = parse_duration_string_ms(non_child_time)
self.__verify_join_time(non_child_time_ms, "Fragment non-child")
check_fragment_count += 1
# Search for "Join Build-Side Prepared Asynchronously"
if ("Join Build-Side Prepared Asynchronously" in line):
asyn_build = True
assert (asyn_build), "Join is not prepared asynchronously: {0}".format(profile)
assert (check_fragment_count > 1), \
"Unable to verify Fragment or Average Fragment: {0}".format(profile)
def _get_json_queries(self, http_addr):
"""Get the json output of the /queries page from the impalad web UI at http_addr."""
if IS_DOCKERIZED_TEST_CLUSTER:
# The hostnames in the dockerized cluster may not be externally reachable.
cluster = ImpalaCluster.get_e2e_test_cluster()
return cluster.impalads[0].service.get_debug_webpage_json("/queries")
else:
resp = urlopen("http://%s/queries?json" % http_addr)
assert resp.msg == 'OK'
return json.loads(resp.read())
def test_execqueryfinstances_timeout(self, vector):
for i in range(3):
ex= self.execute_query_expect_failure(self.client, self.TEST_QUERY)
assert "Exec() rpc failed: Timed out" in str(ex)
verifiers = [MetricVerifier(i.service) for i in
ImpalaCluster.get_e2e_test_cluster().impalads]
for v in verifiers:
v.wait_for_metric("impala-server.num-fragments-in-flight", 0)
v.verify_num_unused_buffers()
def _start_impala_cluster(cls, options, impala_log_dir=os.getenv('LOG_DIR', "/tmp/"),
cluster_size=DEFAULT_CLUSTER_SIZE, num_coordinators=NUM_COORDINATORS,
use_exclusive_coordinators=False, log_level=1,
expected_num_executors=DEFAULT_CLUSTER_SIZE, default_query_options=None):
cls.impala_log_dir = impala_log_dir
# We ignore TEST_START_CLUSTER_ARGS here. Custom cluster tests specifically test that
# certain custom startup arguments work and we want to keep them independent of dev
# environments.
cmd = [os.path.join(IMPALA_HOME, 'bin/start-impala-cluster.py'),
'--cluster_size=%d' % cluster_size,
'--num_coordinators=%d' % num_coordinators,
'--log_dir=%s' % impala_log_dir,
'--log_level=%s' % log_level]
if use_exclusive_coordinators:
cmd.append("--use_exclusive_coordinators")
if pytest.config.option.use_local_catalog:
cmd.append("--impalad_args=--use_local_catalog=1")
cmd.append("--catalogd_args=--catalog_topic_mode=minimal")
if pytest.config.option.pull_incremental_statistics:
cmd.append("--impalad_args=%s --catalogd_args=%s" %
("--pull_incremental_statistics", "--pull_incremental_statistics"))
default_query_option_kvs = []
# Put any defaults first, then any arguments after that so they can override defaults.
if os.environ.get("ERASURE_CODING") == "true":
default_query_option_kvs.append(("allow_erasure_coded_files", "true"))
if default_query_options is not None:
default_query_option_kvs.extend(default_query_options)
# Add the default query options after any arguments. This will override any default
# options set in --impalad_args by design to force tests to pass default_query_options
# into this function directly.
options.append("--impalad_args=--default_query_options={0}".format(
','.join(["{0}={1}".format(k, v) for k, v in default_query_option_kvs])))
logging.info("Starting cluster with command: %s" %
" ".join(pipes.quote(arg) for arg in cmd + options))
try:
check_call(cmd + options, close_fds=True)
finally:
# Failure tests expect cluster to be initialised even if start-impala-cluster fails.
cls.cluster = ImpalaCluster.get_e2e_test_cluster()
statestored = cls.cluster.statestored
if statestored is None:
raise Exception("statestored was not found")
# The number of statestore subscribers is
# cluster_size (# of impalad) + 1 (for catalogd).
expected_subscribers = cluster_size + 1
statestored.service.wait_for_live_subscribers(expected_subscribers, timeout=60)
for impalad in cls.cluster.impalads:
impalad.service.wait_for_num_known_live_backends(expected_num_executors, timeout=60)
def _start_impala_cluster(cls, options, impala_log_dir=os.getenv('LOG_DIR', "/tmp/"),
cluster_size=DEFAULT_CLUSTER_SIZE, num_coordinators=NUM_COORDINATORS,
use_exclusive_coordinators=False, log_level=1,
expected_num_executors=DEFAULT_CLUSTER_SIZE, default_query_options=None):
cls.impala_log_dir = impala_log_dir
# We ignore TEST_START_CLUSTER_ARGS here. Custom cluster tests specifically test that
# certain custom startup arguments work and we want to keep them independent of dev
# environments.
cmd = [os.path.join(IMPALA_HOME, 'bin/start-impala-cluster.py'),
'--cluster_size=%d' % cluster_size,
'--num_coordinators=%d' % num_coordinators,
'--log_dir=%s' % impala_log_dir,
'--log_level=%s' % log_level]
if use_exclusive_coordinators:
cmd.append("--use_exclusive_coordinators")
if pytest.config.option.use_local_catalog:
cmd.append("--impalad_args=--use_local_catalog=1")
cmd.append("--catalogd_args=--catalog_topic_mode=minimal")
if pytest.config.option.pull_incremental_statistics:
cmd.append("--impalad_args=%s --catalogd_args=%s" %
("--pull_incremental_statistics", "--pull_incremental_statistics"))
default_query_option_kvs = []
# Put any defaults first, then any arguments after that so they can override defaults.
if os.environ.get("ERASURE_CODING") == "true":
default_query_option_kvs.append(("allow_erasure_coded_files", "true"))
if default_query_options is not None:
default_query_option_kvs.extend(default_query_options)
# Add the default query options after any arguments. This will override any default
# options set in --impalad_args by design to force tests to pass default_query_options
# into this function directly.
options.append("--impalad_args=--default_query_options={0}".format(
','.join(["{0}={1}".format(k, v) for k, v in default_query_option_kvs])))
logging.info("Starting cluster with command: %s" %
" ".join(pipes.quote(arg) for arg in cmd + options))
try:
check_call(cmd + options, close_fds=True)
finally:
# Failure tests expect cluster to be initialised even if start-impala-cluster fails.
cls.cluster = ImpalaCluster.get_e2e_test_cluster()
statestored = cls.cluster.statestored
if statestored is None:
raise Exception("statestored was not found")
# The number of statestore subscribers is
# cluster_size (# of impalad) + 1 (for catalogd).
expected_subscribers = cluster_size + 1
statestored.service.wait_for_live_subscribers(expected_subscribers, timeout=60)
for impalad in cls.cluster.impalads:
impalad.service.wait_for_num_known_live_backends(expected_num_executors, timeout=60)
def execute_query_verify_metrics(self, query, query_options=None, repeat=1):
for i in range(repeat):
try:
self.execute_query(query, query_options)
except ImpalaBeeswaxException:
pass
verifiers = [MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
for v in verifiers:
v.wait_for_metric("impala-server.num-fragments-in-flight", 0)
v.verify_num_unused_buffers()
def test_pull_stats_profile(self, vector, unique_database):
"""Checks that the frontend profile includes metrics when computing
incremental statistics.
"""
try:
impalad = ImpalaCluster.get_e2e_test_cluster().impalads[0]
client = impalad.service.create_beeswax_client()
create = "create table test like functional.alltypes"
load = "insert into test partition(year, month) select * from functional.alltypes"
insert = """insert into test partition(year=2009, month=1) values
(29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
"10/21/09","4","2009-10-21 03:24:09.600000000")"""
stats_all = "compute incremental stats test"
stats_part = "compute incremental stats test partition (year=2009,month=1)"
# Checks that profile does not have metrics for incremental stats when
# the operation is not 'compute incremental stats'.
self.execute_query_expect_success(client,
"use `%s`" % unique_database)
profile = self.execute_query_expect_success(client,
create).runtime_profile
assert profile.count("StatsFetch") == 0
# Checks that incremental stats metrics are present when 'compute incremental
# stats' is run. Since the table has no stats, expect that no bytes are fetched.
self.execute_query_expect_success(client, load)
profile = self.execute_query_expect_success(
client, stats_all).runtime_profile
assert profile.count("StatsFetch") > 1
assert profile.count("StatsFetch.CompressedBytes: 0") == 1
# Checks that bytes fetched is non-zero since incremental stats are present now
# and should have been fetched.
self.execute_query_expect_success(client, insert)
profile = self.execute_query_expect_success(
client, stats_part).runtime_profile
assert profile.count("StatsFetch") > 1
assert profile.count("StatsFetch.CompressedBytes") == 1
assert profile.count("StatsFetch.CompressedBytes: 0") == 0
# Adds a partition, computes stats, and checks that the metrics in the profile
# reflect the operation.
alter = "alter table test add partition(year=2011, month=1)"
insert_new_partition = """
insert into test partition(year=2011, month=1) values
(29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
"10/21/09","4","2009-10-21 03:24:09.600000000")
"""
self.execute_query_expect_success(client, alter)
self.execute_query_expect_success(client, insert_new_partition)
profile = self.execute_query_expect_success(
client, stats_all).runtime_profile
assert profile.count("StatsFetch.TotalPartitions: 25") == 1
assert profile.count("StatsFetch.NumPartitionsWithStats: 24") == 1
finally:
client.close()
def verify_mem_usage(self, non_zero_peak_metrics):
"""Verifies that the memory used by KRPC is returned to the memtrackers and that
metrics in 'non_zero_peak_metrics' have a peak value > 0.
"""
verifiers = [MemUsageVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
for verifier in verifiers:
for metric_name in ALL_METRICS:
usage = verifier.get_mem_usage_values(metric_name)
assert usage["total"] == 0
if metric_name in non_zero_peak_metrics:
assert usage["peak"] > 0, metric_name
def print_metrics(substring):
"""Prints metrics with the given substring in the name"""
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
print ">" * 80
port = impalad.get_webserver_port()
cert = impalad._get_webserver_certificate_file()
print "connections metrics for impalad at port {0}:".format(port)
debug_info = json.loads(ImpaladService(impalad.hostname, webserver_port=port,
webserver_certificate_file=cert).read_debug_webpage('metrics?json'))
for metric in debug_info['metric_group']['metrics']:
if substring in metric['name']:
print json.dumps(metric, indent=1)
print "<" * 80
def test_insert_mem_limit(self, vector):
if (vector.get_value('table_format').file_format == 'parquet'):
vector.get_value('exec_option')['COMPRESSION_CODEC'] = \
vector.get_value('compression_codec')
self.run_test_case('QueryTest/insert-mem-limit', vector,
multiple_impalad=vector.get_value('exec_option')['sync_ddl'] == 1)
# IMPALA-7023: These queries can linger and use up memory, causing subsequent
# tests to hit memory limits. Wait for some time to allow the query to
# be reclaimed.
verifiers = [MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
for v in verifiers:
v.wait_for_metric("impala-server.num-fragments-in-flight", 0, timeout=60)
def test_failure_in_prepare(self):
# Fail the scan node
verifiers = [MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
self.client.execute("SET DEBUG_ACTION='-1:0:PREPARE:FAIL'");
try:
self.client.execute("SELECT COUNT(*) FROM functional.alltypes")
assert "Query should have thrown an error"
except ImpalaBeeswaxException:
pass
for v in verifiers:
v.wait_for_metric(self.IN_FLIGHT_FRAGMENTS, 0)
def verify_mem_usage(self, non_zero_peak_metrics):
"""Verifies that the memory used by KRPC is returned to the memtrackers and that
metrics in 'non_zero_peak_metrics' have a peak value > 0.
"""
verifiers = [
MemUsageVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads
]
for verifier in verifiers:
for metric_name in ALL_METRICS:
usage = verifier.get_mem_usage_values(metric_name)
assert usage["total"] == 0
if metric_name in non_zero_peak_metrics:
assert usage["peak"] > 0, metric_name
def test_failure_in_prepare(self):
# Fail the scan node
verifiers = [
MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads
]
self.client.execute("SET DEBUG_ACTION='-1:0:PREPARE:FAIL'")
try:
self.client.execute("SELECT COUNT(*) FROM functional.alltypes")
assert "Query should have thrown an error"
except ImpalaBeeswaxException:
pass
for v in verifiers:
v.wait_for_metric(self.IN_FLIGHT_FRAGMENTS, 0)
def test_pull_stats_profile(self, vector, unique_database):
"""Checks that the frontend profile includes metrics when computing
incremental statistics.
"""
try:
impalad = ImpalaCluster.get_e2e_test_cluster().impalads[0]
client = impalad.service.create_beeswax_client()
create = "create table test like functional.alltypes"
load = "insert into test partition(year, month) select * from functional.alltypes"
insert = """insert into test partition(year=2009, month=1) values
(29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
"10/21/09","4","2009-10-21 03:24:09.600000000")"""
stats_all = "compute incremental stats test"
stats_part = "compute incremental stats test partition (year=2009,month=1)"
# Checks that profile does not have metrics for incremental stats when
# the operation is not 'compute incremental stats'.
self.execute_query_expect_success(client, "use `%s`" % unique_database)
profile = self.execute_query_expect_success(client, create).runtime_profile
assert profile.count("StatsFetch") == 0
# Checks that incremental stats metrics are present when 'compute incremental
# stats' is run. Since the table has no stats, expect that no bytes are fetched.
self.execute_query_expect_success(client, load)
profile = self.execute_query_expect_success(client, stats_all).runtime_profile
assert profile.count("StatsFetch") > 1
assert profile.count("StatsFetch.CompressedBytes: 0") == 1
# Checks that bytes fetched is non-zero since incremental stats are present now
# and should have been fetched.
self.execute_query_expect_success(client, insert)
profile = self.execute_query_expect_success(client, stats_part).runtime_profile
assert profile.count("StatsFetch") > 1
assert profile.count("StatsFetch.CompressedBytes") == 1
assert profile.count("StatsFetch.CompressedBytes: 0") == 0
# Adds a partition, computes stats, and checks that the metrics in the profile
# reflect the operation.
alter = "alter table test add partition(year=2011, month=1)"
insert_new_partition = """
insert into test partition(year=2011, month=1) values
(29349999, true, 4, 4, 4, 40,4.400000095367432,40.4,
"10/21/09","4","2009-10-21 03:24:09.600000000")
"""
self.execute_query_expect_success(client, alter)
self.execute_query_expect_success(client, insert_new_partition)
profile = self.execute_query_expect_success(client, stats_all).runtime_profile
assert profile.count("StatsFetch.TotalPartitions: 25") == 1
assert profile.count("StatsFetch.NumPartitionsWithStats: 24") == 1
finally:
client.close()
def execute_query_then_cancel(self, query, vector, repeat = 1):
for _ in range(repeat):
handle = self.execute_query_async(query, vector.get_value('exec_option'))
self.client.fetch(query, handle)
try:
self.client.cancel(handle)
except ImpalaBeeswaxException:
pass
finally:
self.client.close_query(handle)
verifiers = [MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
for v in verifiers:
v.wait_for_metric("impala-server.num-fragments-in-flight", 0)
v.verify_num_unused_buffers()
def test_hive_udfs_missing_jar(self, vector, unique_database):
""" IMPALA-2365: Impalad shouldn't crash if the udf jar isn't present
on HDFS"""
# Copy hive-exec.jar to a temporary file
jar_path = get_fs_path(
"/test-warehouse/{0}.db/".format(unique_database) +
get_random_id(5) + ".jar")
hive_jar = get_fs_path("/test-warehouse/hive-exec.jar")
self.filesystem_client.copy(hive_jar, jar_path)
drop_fn_stmt = ("drop function if exists "
"`{0}`.`pi_missing_jar`()".format(unique_database))
create_fn_stmt = (
"create function `{0}`.`pi_missing_jar`() returns double location '{1}' "
"symbol='org.apache.hadoop.hive.ql.udf.UDFPI'".format(
unique_database, jar_path))
cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = cluster.get_any_impalad()
client = impalad.service.create_beeswax_client()
# Create and drop functions with sync_ddl to make sure they are reflected
# in every impalad.
exec_option = copy(vector.get_value('exec_option'))
exec_option['sync_ddl'] = 1
self.execute_query_expect_success(client, drop_fn_stmt, exec_option)
self.execute_query_expect_success(client, create_fn_stmt, exec_option)
# Delete the udf jar
check_call(["hadoop", "fs", "-rm", jar_path])
different_impalad = cluster.get_different_impalad(impalad)
client = different_impalad.service.create_beeswax_client()
# Run a query using the udf from an impalad other than the one
# we used to create the function. This is to bypass loading from
# the cache
try:
self.execute_query_using_client(
client,
"select `{0}`.`pi_missing_jar`()".format(unique_database),
vector)
assert False, "Query expected to fail"
except ImpalaBeeswaxException, e:
assert "Failed to get file info" in str(e)
def test_table_is_cached(self, vector):
cached_read_metric = "impala-server.io-mgr.cached-bytes-read"
query_string = "select count(*) from tpch.nation"
expected_bytes_delta = 2199
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
# Collect the cached read metric on all the impalads before running the query
cached_bytes_before = list()
for impalad in impala_cluster.impalads:
cached_bytes_before.append(
impalad.service.get_metric_value(cached_read_metric))
# Execute the query.
result = self.execute_query(query_string)
assert (len(result.data) == 1)
assert (result.data[0] == '25')
# Read the metrics again.
cached_bytes_after = list()
for impalad in impala_cluster.impalads:
cached_bytes_after.append(
impalad.service.get_metric_value(cached_read_metric))
# Verify that the cached bytes increased by the expected number on exactly one of
# the impalads.
num_metrics_increased = 0
assert (len(cached_bytes_before) == len(cached_bytes_after))
for i in range(0, len(cached_bytes_before)):
assert(cached_bytes_before[i] == cached_bytes_after[i] or\
cached_bytes_before[i] + expected_bytes_delta == cached_bytes_after[i])
if cached_bytes_after[i] > cached_bytes_before[i]:
num_metrics_increased = num_metrics_increased + 1
if IS_DOCKERIZED_TEST_CLUSTER:
assert num_metrics_increased == 0, "HDFS caching is disabled in dockerised cluster."
elif num_metrics_increased != 1:
# Test failed, print the metrics
for i in range(0, len(cached_bytes_before)):
print "%d %d" % (cached_bytes_before[i], cached_bytes_after[i])
assert (False)
def test_hive_udfs_missing_jar(self, vector, unique_database):
""" IMPALA-2365: Impalad shouldn't crash if the udf jar isn't present
on HDFS"""
# Copy hive-exec.jar to a temporary file
jar_path = get_fs_path("/test-warehouse/{0}.db/".format(unique_database)
+ get_random_id(5) + ".jar")
hive_jar = get_fs_path("/test-warehouse/hive-exec.jar")
check_call(["hadoop", "fs", "-cp", hive_jar, jar_path])
drop_fn_stmt = (
"drop function if exists "
"`{0}`.`pi_missing_jar`()".format(unique_database))
create_fn_stmt = (
"create function `{0}`.`pi_missing_jar`() returns double location '{1}' "
"symbol='org.apache.hadoop.hive.ql.udf.UDFPI'".format(unique_database, jar_path))
cluster = ImpalaCluster.get_e2e_test_cluster()
impalad = cluster.get_any_impalad()
client = impalad.service.create_beeswax_client()
# Create and drop functions with sync_ddl to make sure they are reflected
# in every impalad.
exec_option = copy(vector.get_value('exec_option'))
exec_option['sync_ddl'] = 1
self.execute_query_expect_success(client, drop_fn_stmt, exec_option)
self.execute_query_expect_success(client, create_fn_stmt, exec_option)
# Delete the udf jar
check_call(["hadoop", "fs", "-rm", jar_path])
different_impalad = cluster.get_different_impalad(impalad)
client = different_impalad.service.create_beeswax_client()
# Run a query using the udf from an impalad other than the one
# we used to create the function. This is to bypass loading from
# the cache
try:
self.execute_query_using_client(
client, "select `{0}`.`pi_missing_jar`()".format(unique_database), vector)
assert False, "Query expected to fail"
except ImpalaBeeswaxException, e:
assert "Failed to get file info" in str(e)
def test_table_is_cached(self, vector):
cached_read_metric = "impala-server.io-mgr.cached-bytes-read"
query_string = "select count(*) from tpch.nation"
expected_bytes_delta = 2199
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
# Collect the cached read metric on all the impalads before running the query
cached_bytes_before = list()
for impalad in impala_cluster.impalads:
cached_bytes_before.append(impalad.service.get_metric_value(cached_read_metric))
# Execute the query.
result = self.execute_query(query_string)
assert(len(result.data) == 1)
assert(result.data[0] == '25')
# Read the metrics again.
cached_bytes_after = list()
for impalad in impala_cluster.impalads:
cached_bytes_after.append(impalad.service.get_metric_value(cached_read_metric))
# Verify that the cached bytes increased by the expected number on exactly one of
# the impalads.
num_metrics_increased = 0
assert(len(cached_bytes_before) == len(cached_bytes_after))
for i in range(0, len(cached_bytes_before)):
assert(cached_bytes_before[i] == cached_bytes_after[i] or\
cached_bytes_before[i] + expected_bytes_delta == cached_bytes_after[i])
if cached_bytes_after[i] > cached_bytes_before[i]:
num_metrics_increased = num_metrics_increased + 1
if IS_DOCKERIZED_TEST_CLUSTER:
assert num_metrics_increased == 0, "HDFS caching is disabled in dockerised cluster."
elif num_metrics_increased != 1:
# Test failed, print the metrics
for i in range(0, len(cached_bytes_before)):
print "%d %d" % (cached_bytes_before[i], cached_bytes_after[i])
assert(False)
def wait_for_event_processing(test_suite, timeout=10):
"""Waits till the event processor has synced to the latest event id from metastore
or the timeout value in seconds whichever is earlier"""
if EventProcessorUtils.get_event_processor_status() == "DISABLED":
return
success = False
assert timeout > 0
assert test_suite.hive_client is not None
current_event_id = EventProcessorUtils.get_current_notification_id(
test_suite.hive_client)
LOG.info("Waiting until events processor syncs to event id:" +
str(current_event_id))
end_time = time.time() + timeout
while time.time() < end_time:
last_synced_id = EventProcessorUtils.get_last_synced_event_id()
if last_synced_id >= current_event_id:
LOG.debug(
"Metric last-synced-event-id has reached the desired value:"
+ str(last_synced_id))
success = True
break
time.sleep(0.1)
if not success:
raise Exception(
"Event processor did not sync till last known event id {0} \
within {1} seconds".format(current_event_id, timeout))
if isinstance(test_suite, CustomClusterTestSuite):
impala_cluster = test_suite.cluster
else:
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
# Wait until the impalad catalog versions agree with the catalogd's version.
catalogd_version = impala_cluster.catalogd.service.get_catalog_version(
)
for impalad in impala_cluster.impalads:
impalad.service.wait_for_metric_value("catalog.curr-version",
catalogd_version,
allow_greater=True)
return success
def test_failure_in_prepare_multi_fragment(self):
# Test that if one fragment fails that the others are cleaned up during the ensuing
# cancellation.
verifiers = [MetricVerifier(i.service)
for i in ImpalaCluster.get_e2e_test_cluster().impalads]
# Fail the scan node
self.client.execute("SET DEBUG_ACTION='-1:0:PREPARE:FAIL'");
# Force a query plan that will have three fragments or more.
try:
self.client.execute("SELECT COUNT(*) FROM functional.alltypes a JOIN [SHUFFLE] \
functional.alltypes b on a.id = b.id")
assert "Query should have thrown an error"
except ImpalaBeeswaxException:
pass
for v in verifiers:
# Long timeout required because fragments may be blocked while sending data. The
# default value of --datastream_sender_timeout_ms is 120s before they wake up and
# cancel themselves.
#
# TODO: Fix when we have cancellable RPCs.
v.wait_for_metric(self.IN_FLIGHT_FRAGMENTS, 0, timeout=125)
def test_views_describe(self, vector, unique_database):
# IMPALA-6896: Tests that altered views can be described by all impalads.
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
impalads = impala_cluster.impalads
view_name = "%s.test_describe_view" % unique_database
query_opts = vector.get_value('exec_option')
first_client = impalads[0].service.create_beeswax_client()
try:
# Create a view and verify it's visible.
self.execute_query_expect_success(first_client,
"create view {0} as "
"select * from functional.alltypes"
.format(view_name), query_opts)
self._verify_describe_view(vector, view_name, "select * from functional.alltypes")
# Alter the view and verify the alter is visible.
self.execute_query_expect_success(first_client,
"alter view {0} as "
"select * from functional.alltypesagg"
.format(view_name), query_opts)
self._verify_describe_view(vector, view_name,
"select * from functional.alltypesagg")
finally:
first_client.close()
try:
self.run_test_case('QueryTest/udf-mem-limit', vector, use_db=unique_database)
assert False, "Query was expected to fail"
except ImpalaBeeswaxException, e:
self._check_mem_limit_exception(e)
try:
self.run_test_case('QueryTest/uda-mem-limit', vector, use_db=unique_database)
assert False, "Query was expected to fail"
except ImpalaBeeswaxException, e:
self._check_mem_limit_exception(e)
# It takes a long time for Impala to free up memory after this test, especially if
# ASAN is enabled. Verify that all fragments finish executing before moving on to the
# next test to make sure that the next test is not affected.
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0)
verifier.verify_num_unused_buffers()
def test_udf_constant_folding(self, vector, unique_database):
"""Test that constant folding of UDFs is handled correctly. Uses count_rows(),
which returns a unique value every time it is evaluated in the same thread."""
exec_options = copy(vector.get_value('exec_option'))
# Execute on a single node so that all counter values will be unique.
exec_options["num_nodes"] = 1
create_fn_query = """create function {database}.count_rows() returns bigint
location '{location}' symbol='Count' prepare_fn='CountPrepare'
close_fn='CountClose'"""
self._load_functions(create_fn_query, vector, unique_database,
get_fs_path('/test-warehouse/libTestUdfs.so'))
def _run_query_all_impalads(self, exec_options, query, expected):
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
for impalad in impala_cluster.impalads:
client = impalad.service.create_beeswax_client()
result = self.execute_query_expect_success(client, query, exec_options)
assert result.data == expected
assert False, "Query was expected to fail"
except ImpalaBeeswaxException, e:
self._check_mem_limit_exception(e)
try:
self.run_test_case('QueryTest/uda-mem-limit',
vector,
use_db=unique_database)
assert False, "Query was expected to fail"
except ImpalaBeeswaxException, e:
self._check_mem_limit_exception(e)
# It takes a long time for Impala to free up memory after this test, especially if
# ASAN is enabled. Verify that all fragments finish executing before moving on to the
# next test to make sure that the next test is not affected.
for impalad in ImpalaCluster.get_e2e_test_cluster().impalads:
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight",
0)
verifier.verify_num_unused_buffers()
def test_udf_constant_folding(self, vector, unique_database):
"""Test that constant folding of UDFs is handled correctly. Uses count_rows(),
which returns a unique value every time it is evaluated in the same thread."""
exec_options = copy(vector.get_value('exec_option'))
# Execute on a single node so that all counter values will be unique.
exec_options["num_nodes"] = 1
create_fn_query = """create function {database}.count_rows() returns bigint
location '{location}' symbol='Count' prepare_fn='CountPrepare'
close_fn='CountClose'"""
self._load_functions(create_fn_query, vector, unique_database,
def _get_json_queries(self, http_addr):
"""Get the json output of the /queries page from the impalad web UI at http_addr."""
cluster = ImpalaCluster.get_e2e_test_cluster()
return cluster.impalads[0].service.get_debug_webpage_json("/queries")
def test_native_functions_race(self, vector, unique_database):
""" IMPALA-6488: stress concurrent adds, uses, and deletes of native functions.
Exposes a crash caused by use-after-free in lib-cache."""
# Native function used by a query. Stresses lib-cache during analysis and
# backend expressions.
create_fn_to_use = \
"""create function {0}.use_it(string) returns string
LOCATION '{1}'
SYMBOL='_Z8IdentityPN10impala_udf15FunctionContextERKNS_9StringValE'"""
use_fn = """select * from (select max(int_col) from functional.alltypesagg
where {0}.use_it(string_col) = 'blah' union all
(select max(int_col) from functional.alltypesagg
where {0}.use_it(String_col) > '1' union all
(select max(int_col) from functional.alltypesagg
where {0}.use_it(string_col) > '1'))) v"""
# Reference to another native function from the same 'so' file. Creating/dropping
# stresses lib-cache lookup, add, and refresh.
create_another_fn = """create function if not exists {0}.other(float)
returns float location '{1}' symbol='Identity'"""
drop_another_fn = """drop function if exists {0}.other(float)"""
udf_path = get_fs_path('/test-warehouse/libTestUdfs.so')
# Tracks number of impalads prior to tests to check that none have crashed.
# All impalads are assumed to be coordinators.
cluster = ImpalaCluster.get_e2e_test_cluster()
exp_num_coordinators = cluster.num_responsive_coordinators()
setup_client = self.create_impala_client()
setup_query = create_fn_to_use.format(unique_database, udf_path)
try:
setup_client.execute(setup_query)
except Exception as e:
print "Unable to create initial function: {0}".format(setup_query)
raise
errors = []
def use_fn_method():
time.sleep(1 + random.random())
client = self.create_impala_client()
query = use_fn.format(unique_database)
try:
client.execute(query)
except Exception as e:
errors.append(e)
def load_fn_method():
time.sleep(1 + random.random())
client = self.create_impala_client()
drop = drop_another_fn.format(unique_database)
create = create_another_fn.format(unique_database, udf_path)
try:
client.execute(drop)
client.execute(create)
except Exception as e:
errors.append(e)
# number of uses/loads needed to reliably reproduce the bug.
num_uses = 200
num_loads = 200
# create threads to use native function.
runner_threads = []
for i in xrange(num_uses):
runner_threads.append(threading.Thread(target=use_fn_method))
# create threads to drop/create native functions.
for i in xrange(num_loads):
runner_threads.append(threading.Thread(target=load_fn_method))
# launch all runner threads.
for t in runner_threads: t.start()
# join all threads.
for t in runner_threads: t.join()
for e in errors: print e
# Checks that no impalad has crashed.
assert cluster.num_responsive_coordinators() == exp_num_coordinators
def test_get_functions(self, vector, unique_database):
impala_cluster = ImpalaCluster.get_e2e_test_cluster()
catalogd = impala_cluster.catalogd.service
trans_type = 'buffered'
if pytest.config.option.use_kerberos:
trans_type = 'kerberos'
transport = create_transport(host=catalogd.hostname, port=catalogd.service_port,
service='impala', transport_type=trans_type)
transport.open()
protocol = TBinaryProtocol.TBinaryProtocol(transport)
catalog_client = CatalogService.Client(protocol)
request = TGetFunctionsRequest()
request.db_name = unique_database
response = catalog_client.GetFunctions(request)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 0
self.client.execute("create function %s.fn() RETURNS int "
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'"
% (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert len(response.functions) == 1
assert len(response.functions[0].arg_types) == 0
assert response.functions[0].name.db_name == unique_database
assert response.functions[0].name.function_name == 'fn'
assert response.functions[0].aggregate_fn is None
assert response.functions[0].scalar_fn is not None
assert '/test-warehouse/libTestUdfs.so' in response.functions[0].hdfs_location
# Add another scalar function with overloaded parameters ensure it shows up.
self.client.execute("create function %s.fn(int) RETURNS double "\
"LOCATION '%s/libTestUdfs.so' SYMBOL='Fn'" % (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 2
functions = [fn for fn in response.functions]
# Sort by number of arg in the function (ascending)
functions.sort(key=lambda fn: len(fn.arg_types))
assert len(functions[0].arg_types) == 0
assert len(functions[1].arg_types) == 1
assert functions[0].signature == 'fn()'
assert functions[1].signature == 'fn(INT)'
# Verify aggregate functions can also be retrieved
self.client.execute("create aggregate function %s.agg_fn(int, string) RETURNS int "
"LOCATION '%s/libTestUdas.so' UPDATE_FN='TwoArgUpdate'"
% (unique_database, WAREHOUSE))
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.OK
assert len(response.functions) == 3
functions = [fn for fn in response.functions if fn.aggregate_fn is not None]
# Should be only 1 aggregate function
assert len(functions) == 1
# Negative test cases for database name
request.db_name = unique_database + "_does_not_exist"
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database does not exist: ' in str(response.status)
request = TGetFunctionsRequest()
response = catalog_client.GetFunctions(request)
LOG.debug(response)
assert response.status.status_code == TErrorCode.GENERAL
assert 'Database name must be set' in str(response.status)
