def add_test_dimensions(cls):
super(TestAdmissionControllerStress, cls).add_test_dimensions()
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('num_queries', *NUM_QUERIES))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('round_robin_submission',
*ROUND_ROBIN_SUBMISSION))
cls.ImpalaTestMatrix.add_dimension(
ImpalaTestDimension('submission_delay_ms', *SUBMISSION_DELAY_MS))
# Additional constraints for code coverage jobs and core.
num_queries = None
if IMPALAD_BUILD.has_code_coverage():
# Code coverage builds can't handle the increased concurrency.
num_queries = 15
elif cls.exploration_strategy() == 'core':
num_queries = 30
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('submission_delay_ms') == 0)
cls.ImpalaTestMatrix.add_constraint(\
lambda v: v.get_value('round_robin_submission') == True)
if num_queries is not None:
cls.ImpalaTestMatrix.add_constraint(
lambda v: v.get_value('num_queries') == num_queries)
class TestAutomaticCatalogInvalidation(CustomClusterTestSuite):
""" Test that tables are cached in the catalogd after usage for the configured time
and invalidated afterwards."""
query = "select count(*) from functional.alltypes"
# The following columns string presents in the catalog object iff the table loaded.
metadata_cache_string = "columns (list) = list<struct>"
url = "http://localhost:25020/catalog_object?object_type=TABLE&" \
"object_name=functional.alltypes"
timeout_flag = "--invalidate_tables_timeout_s=" + \
("20" if IMPALAD_BUILD.runs_slowly() or (not IS_HDFS and not IS_LOCAL) else "10")
@classmethod
def get_workload(cls):
return 'functional-query'
def _get_catalog_object(self):
""" Return the catalog object of functional.alltypes serialized to string. """
return self.cluster.catalogd.service.read_debug_webpage(
"catalog_object?object_type=TABLE&object_name=functional.alltypes")
def _run_test(self, cursor):
cursor.execute(self.query)
cursor.fetchall()
# The table is cached after usage.
assert self.metadata_cache_string in self._get_catalog_object()
timeout = time.time() + 20
while True:
time.sleep(1)
# The table is eventually evicted.
if self.metadata_cache_string not in self._get_catalog_object():
return
assert time.time() < timeout
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(catalogd_args=timeout_flag, impalad_args=timeout_flag)
def test_v1_catalog(self, cursor):
self._run_test(cursor)
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args=timeout_flag + " --catalog_topic_mode=minimal",
impalad_args=timeout_flag + " --use_local_catalog")
def test_local_catalog(self, cursor):
self._run_test(cursor)
def add_test_dimensions(cls):
super(TestAdmissionControllerStress, cls).add_test_dimensions()
cls.TestMatrix.add_dimension(TestDimension('num_queries', *NUM_QUERIES))
cls.TestMatrix.add_dimension(
TestDimension('round_robin_submission', *ROUND_ROBIN_SUBMISSION))
cls.TestMatrix.add_dimension(
TestDimension('submission_delay_ms', *SUBMISSION_DELAY_MS))
# Additional constraints for code coverage jobs and core.
num_queries = None
if IMPALAD_BUILD.has_code_coverage():
# Code coverage builds can't handle the increased concurrency.
num_queries = 15
elif cls.exploration_strategy() == 'core':
num_queries = 30
cls.TestMatrix.add_constraint(lambda v: v.get_value('submission_delay_ms') == 0)
cls.TestMatrix.add_constraint(\
lambda v: v.get_value('round_robin_submission') == True)
if num_queries is not None:
cls.TestMatrix.add_constraint(lambda v: v.get_value('num_queries') == num_queries)
class SkipIfBuildType:
not_dev_build = pytest.mark.skipif(
not IMPALAD_BUILD.is_dev(),
reason="Tests depends on debug build startup option.")
class TestAutomaticCatalogInvalidation(CustomClusterTestSuite):
""" Test that tables are cached in the catalogd after usage for the configured time
and invalidated afterwards."""
query = "select count(*) from functional.alltypes"
# The following columns string presents in the catalog object iff the table loaded.
metadata_cache_string = "columns (list) = list<struct>"
url = "http://localhost:25020/catalog_object?object_type=TABLE&" \
"object_name=functional.alltypes"
# The test will run a query and assumes the table is loaded when the query finishes.
# The timeout should be larger than the time of the query.
timeout = 20 if IMPALAD_BUILD.runs_slowly() or (not IS_HDFS
and not IS_LOCAL) else 10
timeout_flag = "--invalidate_tables_timeout_s=" + str(timeout)
@classmethod
def get_workload(cls):
return 'functional-query'
def _get_catalog_object(self):
""" Return the catalog object of functional.alltypes serialized to string. """
return self.cluster.catalogd.service.read_debug_webpage(
"catalog_object?object_type=TABLE&object_name=functional.alltypes")
def _run_test(self, cursor):
cursor.execute(self.query)
cursor.fetchall()
# The table is cached after usage.
assert self.metadata_cache_string in self._get_catalog_object()
# Wait 2 * table TTL for the invalidation to take effect.
max_wait_time = time.time() + self.timeout * 2
while True:
time.sleep(1)
# The table is eventually evicted.
if self.metadata_cache_string not in self._get_catalog_object():
return
assert time.time() < max_wait_time
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(catalogd_args=timeout_flag,
impalad_args=timeout_flag)
def test_v1_catalog(self, cursor):
self._run_test(cursor)
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args=timeout_flag + " --catalog_topic_mode=minimal",
impalad_args=timeout_flag + " --use_local_catalog")
def test_local_catalog(self, cursor):
self._run_test(cursor)
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args="--invalidate_tables_timeout_s=1",
impalad_args="--invalidate_tables_timeout_s=1")
def test_invalid_table(self):
""" Regression test for IMPALA-7606. Tables failed to be loaded don't have a
last used time and shouldn't be considered for invalidation."""
self.execute_query_expect_failure(
self.client, "select * from functional.bad_serde")
# The table expires after 1 second. Sleeping for another logbufsecs=5 seconds to wait
# for the log to be flushed. Wait 4 more seconds to reduce flakiness.
time.sleep(10)
assert "Unexpected exception thrown while attempting to automatically invalidate "\
"tables" not in open(os.path.join(self.impala_log_dir, "catalogd.INFO")).read()
@pytest.mark.execute_serially
@CustomClusterTestSuite.with_args(
catalogd_args="--invalidate_tables_on_memory_pressure "
"--invalidate_tables_gc_old_gen_full_threshold=0 "
"--invalidate_tables_fraction_on_memory_pressure=1",
impalad_args="--invalidate_tables_on_memory_pressure")
def test_memory_pressure(self):
""" Test that memory-based invalidation kicks out all the tables after an GC."""
self.execute_query(self.query)
# This triggers a full GC as of openjdk 1.8.
call(["jmap", "-histo:live", str(self.cluster.catalogd.get_pid())])
# Sleep for logbufsecs=5 seconds to wait for the log to be flushed. Wait 5 more
# seconds to reduce flakiness.
time.sleep(10)
assert self.metadata_cache_string not in self._get_catalog_object()